RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support per-request download handlers

This commit is contained in:
Adrian Chaves 2026-08-05 06:21:49 +02:00
parent 91b70e4db4
commit cdabf42584
10 changed files with 287 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
docs/topics/download-handlers.rst
View File

@ -43,6 +43,44 @@ for additional schemes and to replace or disable default ones:
:ref:`security-local-resources`, for the security implications of the
default ``http``, ``ftp``, ``file`` and ``data`` handlers.
.. _download-handler-ids:
Choosing a download handler per request
---------------------------------------
.. versionadded:: VERSION
The :setting:`DOWNLOAD_HANDLERS_BY_NAME` setting registers handlers under a
name instead of a URL scheme. Such handlers are only used by requests that ask
for them through the :reqmeta:`download_handler` metadata key:
.. code-block:: python
DOWNLOAD_HANDLERS_BY_NAME = {
"playwright": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
}
.. code-block:: python
Request("https://example.com", meta={"download_handler": "playwright"})
:reqmeta:`download_handler` also accepts a URL scheme, e.g. ``"https"``, to use
a handler from :setting:`DOWNLOAD_HANDLERS` on a request whose URL uses a
different scheme.
Requests keep their :reqmeta:`download_handler` across redirects and retries,
and :ref:`robots.txt <topics-dlmw-robots>` requests inherit it from the request
that triggers them. To fetch ``robots.txt`` differently, use a :ref:`downloader
middleware <topics-downloader-middleware>` that acts on requests with the
:reqmeta:`is_robotstxt_request` metadata key:
.. code-block:: python
class DirectRobotsTxtMiddleware:
def process_request(self, request):
if request.meta.get("is_robotstxt_request"):
request.meta.pop("download_handler", None)
Replacing HTTP(S) download handlers
-----------------------------------

23
docs/topics/request-response.rst
View File

@ -943,6 +943,17 @@ This meta key is not supported by
:class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`, but the
:setting:`DOWNLOAD_BIND_ADDRESS` is supported by it.
.. reqmeta:: download_handler
download_handler
----------------
.. versionadded:: VERSION
ID of the :ref:`download handler <topics-download-handlers>` to use for this
request, either a name from :setting:`DOWNLOAD_HANDLERS_BY_NAME` or a URL
scheme from :setting:`DOWNLOAD_HANDLERS`. See :ref:`download-handler-ids`.
.. reqmeta:: download_timeout
download_timeout
@ -1006,6 +1017,18 @@ http_user
Overrides :setting:`HTTPAUTH_USER` for this request.
.. reqmeta:: is_robotstxt_request
is_robotstxt_request
--------------------
.. versionadded:: VERSION
``True`` on the ``robots.txt`` requests that
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` sends, so
that :ref:`downloader middlewares <topics-downloader-middleware>` can tell them
apart from other requests.
.. reqmeta:: max_retry_times
max_retry_times

13
docs/topics/settings.rst
View File

@ -1062,6 +1062,19 @@ handler (without replacement), place this in your ``settings.py``:
:ref:`security-local-resources`
.. setting:: DOWNLOAD_HANDLERS_BY_NAME
DOWNLOAD_HANDLERS_BY_NAME
-------------------------
.. versionadded:: VERSION
Default: ``{}``
A dict mapping names to :ref:`download handlers <topics-download-handlers>`
that requests can ask for by name. See :ref:`download-handler-ids`.
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS

96
scrapy/core/downloader/handlers/__init__.py
View File

@ -61,44 +61,59 @@ class DownloadHandlerProtocol(Protocol):
class DownloadHandlers:
def __init__(self, crawler: Crawler):
self._crawler: Crawler = crawler
# stores acceptable schemes on instancing
self._schemes: dict[str, str | Callable[..., Any]] = {}
# stores instanced handlers for schemes
# stores class paths by handler ID
self._paths: dict[str, str | Callable[..., Any]] = {}
# handler IDs that are also acceptable URL schemes
self._schemes: set[str] = set()
# stores instanced handlers by handler ID
self._handlers: dict[str, DownloadHandlerProtocol] = {}
# remembers failed handlers
self._notconfigured: dict[str, str] = {}
# remembers handlers with Deferred-based download_request()
self._old_style_handlers: set[str] = set()
handlers: dict[str, str | Callable[..., Any]] = without_none_values(
scheme_handlers: dict[str, str | Callable[..., Any]] = without_none_values(
cast(
"dict[str, str | Callable[..., Any]]",
crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
)
)
for scheme, clspath in handlers.items():
self._schemes[scheme] = clspath
self._load_handler(scheme, skip_lazy=True)
named_handlers: dict[str, str | Callable[..., Any]] = without_none_values(
cast(
"dict[str, str | Callable[..., Any]]",
crawler.settings.getdict("DOWNLOAD_HANDLERS_BY_NAME"),
)
)
if clashes := scheme_handlers.keys() & named_handlers.keys():
raise ValueError(
f"The following download handler IDs are defined both in "
f"DOWNLOAD_HANDLERS and in DOWNLOAD_HANDLERS_BY_NAME: "
f"{', '.join(sorted(clashes))}."
)
self._schemes.update(scheme_handlers)
for handler_id, clspath in (scheme_handlers | named_handlers).items():
self._paths[handler_id] = clspath
self._load_handler(handler_id, skip_lazy=True)
crawler.signals.connect(self._close, signals.engine_stopped)
def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None:
"""Lazy-load the downloadhandler for a scheme
only on the first request for that scheme.
def _get_handler(self, handler_id: str) -> DownloadHandlerProtocol | None:
"""Lazy-load the download handler with the given ID only on its first
request.
"""
if scheme in self._handlers:
return self._handlers[scheme]
if scheme in self._notconfigured:
if handler_id in self._handlers:
return self._handlers[handler_id]
if handler_id in self._notconfigured:
return None
if scheme not in self._schemes:
self._notconfigured[scheme] = "no handler available for that scheme"
if handler_id not in self._paths:
self._notconfigured[handler_id] = "no handler with that ID"
return None
return self._load_handler(scheme)
return self._load_handler(handler_id)
def _load_handler(
self, scheme: str, skip_lazy: bool = False
self, handler_id: str, skip_lazy: bool = False
) -> DownloadHandlerProtocol | None:
path = self._schemes[scheme]
path = self._paths[handler_id]
try:
dhcls: type[DownloadHandlerProtocol] = load_object(path)
if skip_lazy:
@ -117,18 +132,18 @@ class DownloadHandlers:
self._crawler,
)
except NotConfigured as ex:
self._notconfigured[scheme] = str(ex)
self._notconfigured[handler_id] = str(ex)
return None
except Exception as ex:
logger.error(
'Loading "%(clspath)s" for scheme "%(scheme)s"',
{"clspath": path, "scheme": scheme},
'Loading "%(clspath)s" for handler ID "%(handler_id)s"',
{"clspath": path, "handler_id": handler_id},
exc_info=True,
extra={"crawler": self._crawler},
)
self._notconfigured[scheme] = str(ex)
self._notconfigured[handler_id] = str(ex)
return None
self._handlers[scheme] = dh
self._handlers[handler_id] = dh
if not inspect.iscoroutinefunction(dh.download_request): # pragma: no cover
warnings.warn(
f"{global_object_name(dh.download_request)} is not a coroutine function."
@ -137,7 +152,7 @@ class DownloadHandlers:
category=ScrapyDeprecationWarning,
stacklevel=1,
)
self._old_style_handlers.add(scheme)
self._old_style_handlers.add(handler_id)
return dh
def download_request(
@ -151,14 +166,33 @@ class DownloadHandlers:
return deferred_from_coro(self.download_request_async(request))
async def download_request_async(self, request: Request) -> Response:
scheme = urlparse_cached(request).scheme
handler = self._get_handler(scheme)
if not handler:
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
handler_id = request.meta.get("download_handler")
if handler_id is None:
scheme = urlparse_cached(request).scheme
# Handler IDs from DOWNLOAD_HANDLERS_BY_NAME are only reachable
# through the download_handler request metadata key, so that
# registering one does not make a new URL scheme downloadable.
if scheme in self._schemes:
handler = self._get_handler(scheme)
else:
handler = None
self._notconfigured.setdefault(
scheme, "no handler available for that scheme"
)
if not handler:
raise NotSupported(
f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
)
handler_id = scheme
else:
handler = self._get_handler(handler_id)
if not handler:
raise NotSupported(
f"Unusable download handler {handler_id!r}: "
f"{self._notconfigured[handler_id]}"
)
assert self._crawler.spider
if scheme in self._old_style_handlers: # pragma: no cover
if handler_id in self._old_style_handlers: # pragma: no cover
return await maybe_deferred_to_future(
cast(
"Deferred[Response]",

10
scrapy/downloadermiddlewares/robotstxt.py
View File

@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
from __future__ import annotations
import logging
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, Any
from twisted.internet.defer import Deferred
@ -89,10 +89,16 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers:
self._parsers[netloc] = Deferred()
robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
meta: dict[str, Any] = {
"dont_obey_robotstxt": True,
"is_robotstxt_request": True,
}
if "download_handler" in request.meta:
meta["download_handler"] = request.meta["download_handler"]
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
meta={"dont_obey_robotstxt": True},
meta=meta,
callback=NO_CALLBACK,
)
assert self.crawler.engine

2
scrapy/settings/default_settings.py
View File

@ -71,6 +71,7 @@ __all__ = [
"DOWNLOAD_FAIL_ON_DATALOSS",
"DOWNLOAD_HANDLERS",
"DOWNLOAD_HANDLERS_BASE",
"DOWNLOAD_HANDLERS_BY_NAME",
"DOWNLOAD_MAXSIZE",
"DOWNLOAD_SLOTS",
"DOWNLOAD_TIMEOUT",
@ -292,6 +293,7 @@ DOWNLOAD_HANDLERS_BASE = {
"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
DOWNLOAD_HANDLERS_BY_NAME = {}
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m

24
scrapy/utils/request.py
View File

@ -32,7 +32,7 @@ logger = logging.getLogger(__name__)
_fingerprint_cache: WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes]
] = WeakKeyDictionary()
@ -77,8 +77,18 @@ def fingerprint(
)
verbatim_url = bool(request.meta.get("verbatim_url"))
effective_keep_fragments = keep_fragments and not verbatim_url
# A handler ID matching the URL scheme is the one that would be used
# anyway, so it is left out to keep such fingerprints unchanged.
handler_id: str | None = request.meta.get("download_handler")
if handler_id == urlparse_cached(request).scheme:
handler_id = None
cache = _fingerprint_cache.setdefault(request, {})
cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)
cache_key = (
processed_include_headers,
effective_keep_fragments,
verbatim_url,
handler_id,
)
if cache_key not in cache:
# To decode bytes reliably (JSON does not support bytes), regardless of
# character encoding, we use bytes.hex()
@ -100,6 +110,8 @@ def fingerprint(
"body": (request.body or b"").hex(),
"headers": headers,
}
if handler_id is not None:
fingerprint_data["download_handler"] = handler_id
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
cache[cache_key] = hashlib.sha1( # noqa: S324
fingerprint_json.encode()
@ -114,13 +126,17 @@ class RequestFingerprinterProtocol(Protocol):
class RequestFingerprinter:
"""Default fingerprinter.
.. versionchanged:: VERSION
:reqmeta:`download_handler` is taken into account.
It takes into account a canonical version
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
<scrapy.Request.url>` and the values of :attr:`request.method
<scrapy.Request.method>` and :attr:`request.body
<scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that
request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_
hash.
request. It also takes into account :reqmeta:`download_handler` when it
does not match the URL scheme. It then generates an `SHA1
<https://en.wikipedia.org/wiki/SHA-1>`_ hash.
"""
@classmethod

72
tests/test_downloader_handlers.py
View File

@ -15,8 +15,8 @@ from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request, TextResponse
from scrapy.exceptions import NotConfigured, NotSupported, ScrapyDeprecationWarning
from scrapy.http import Request, Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils.boto import is_botocore_available
from scrapy.utils.misc import build_from_crawler
@ -31,6 +31,20 @@ class DummyDH:
pass
class NamedDH:
lazy = False
async def download_request(self, request):
return Response(request.url, body=b"named")
class SchemeDH:
lazy = False
async def download_request(self, request):
return Response(request.url, body=b"scheme")
class DummyLazyDH:
# Default (but deprecated) is lazy for backward compatibility
async def download_request(self, request):
@ -84,7 +98,7 @@ class TestLoad:
assert "scheme" not in dh._handlers
assert "scheme" in dh._notconfigured
assert (
'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for scheme "scheme"'
'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for handler ID "scheme"'
in caplog.text
)
@ -115,6 +129,58 @@ class TestLoad:
assert "scheme" not in dh._notconfigured
class TestHandlerID:
@staticmethod
def _get_dh() -> DownloadHandlers:
crawler = get_crawler(
settings_dict={
"DOWNLOAD_HANDLERS": {"https": SchemeDH},
"DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH},
}
)
crawler.spider = crawler._create_spider()
return DownloadHandlers(crawler)
def test_clashing_ids(self) -> None:
crawler = get_crawler(
settings_dict={
"DOWNLOAD_HANDLERS": {"https": SchemeDH},
"DOWNLOAD_HANDLERS_BY_NAME": {"https": NamedDH},
}
)
with pytest.raises(ValueError, match="defined both in DOWNLOAD_HANDLERS"):
DownloadHandlers(crawler)
@coroutine_test
async def test_name_not_a_scheme(self) -> None:
dh = self._get_dh()
with pytest.raises(NotSupported, match="Unsupported URL scheme 'named'"):
await dh.download_request_async(Request("named://example.com"))
@coroutine_test
async def test_name_in_meta(self) -> None:
dh = self._get_dh()
request = Request("https://example.com", meta={"download_handler": "named"})
response = await dh.download_request_async(request)
assert response.body == b"named"
@coroutine_test
async def test_scheme_in_meta(self) -> None:
dh = self._get_dh()
request = Request("ftp://example.com", meta={"download_handler": "https"})
response = await dh.download_request_async(request)
assert response.body == b"scheme"
@coroutine_test
async def test_unknown_id_in_meta(self) -> None:
dh = self._get_dh()
request = Request("https://example.com", meta={"download_handler": "nmaed"})
with pytest.raises(
NotSupported, match="Unusable download handler 'nmaed': no handler with"
):
await dh.download_request_async(request)
class TestFile:
def setup_method(self):
# add a special char to check that they are handled correctly

11
tests/test_downloadermiddleware_robotstxt.py
View File

@ -245,6 +245,17 @@ Disallow: /some/randome/page.html
await middleware.process_request(Request("http://site.local/allowed"))
assert middleware.process_request_2.called
@coroutine_test
async def test_robotstxt_download_handler(self) -> None:
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
await self.assertNotIgnored(
Request("http://site.local/allowed", meta={"download_handler": "named"}),
middleware,
)
robotsreq = self.crawler.engine.download_async.call_args_list[0][0][0]
assert robotsreq.meta["download_handler"] == "named"
assert robotsreq.meta["is_robotstxt_request"] is True
async def assertNotIgnored(
self, request: Request, middleware: RobotsTxtMiddleware
) -> None:

41
tests/test_utils_request.py
View File

@ -73,13 +73,15 @@ class TestFingerprint:
function: _FingerprintFunction = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
Request,
dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes],
]
| WeakKeyDictionary[
Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]
Request,
dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], str],
]
) = _fingerprint_cache
default_cache_key = (None, False, False)
default_cache_key = (None, False, False, None)
known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
(
Request("http://example.org"),
@ -235,6 +237,39 @@ class TestFingerprint:
assert self.function(r5) == self.function(r5, keep_fragments=True)
assert self.function(r5) == self.function(r5, keep_fragments=False)
def test_download_handler(self):
r1 = Request("https://example.com")
r2 = Request("https://example.com", meta={"download_handler": "playwright"})
r3 = Request("https://example.com", meta={"download_handler": "http"})
assert self.function(r1) != self.function(r2)
assert self.function(r2) != self.function(r3)
# An ID matching the URL scheme is the handler that would be used
# anyway, so it does not change the fingerprint.
r4 = Request("https://example.com", meta={"download_handler": "https"})
assert self.function(r1) == self.function(r4)
# IDs are case-sensitive, and URL schemes are lowercased.
r5 = Request("HTTPS://example.com", meta={"download_handler": "HTTPS"})
assert self.function(r1) != self.function(r5)
def test_download_handler_caching(self):
# The cached fingerprint must account for the handler ID, which can
# change after the first call.
r1 = Request("https://example.com")
fp1 = self.function(r1)
r1.meta["download_handler"] = "playwright"
assert self.function(r1) != fp1
def test_download_handler_and_verbatim_url(self):
r1 = Request(
"https://example.com/a b",
meta={"verbatim_url": True, "download_handler": "playwright"},
)
r2 = Request("https://example.com/a b", meta={"verbatim_url": True})
r3 = Request("https://example.com/a b", meta={"download_handler": "playwright"})
assert len({self.function(r1), self.function(r2), self.function(r3)}) == 3
def test_method_and_body(self):
r1 = Request("http://www.example.com")
r2 = Request("http://www.example.com", method="POST")