diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index fcfe7fd29..0e7516a17 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -169,6 +169,42 @@ middleware, see the :ref:`downloader middleware usage guide For a list of the components enabled by default (and their orders) see the :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting. +ChecksumMiddleware +------------------ + +.. autoclass:: scrapy.downloadermiddlewares.checksum.ChecksumMiddleware + +.. autoexception:: scrapy.exceptions.ChecksumError + +.. reqmeta:: expected_checksum + +expected_checksum +~~~~~~~~~~~~~~~~~ + +.. versionadded:: VERSION + +A dict that maps :mod:`hashlib` algorithm names to the expected checksum of the +response body, as :class:`bytes` or as a hexadecimal :class:`str`: + +.. invisible-code-block: python + + from scrapy import Request + +.. code-block:: python + + Request( + "https://example.com/product1.pdf", + meta={ + "expected_checksum": { + "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + } + }, + ) + +A mismatch is retried like any other download failure, and once retries are +exhausted the request fails with :exc:`~scrapy.exceptions.ChecksumError`. Set +the :reqmeta:`dont_retry` meta key to fail on the first mismatch. + .. _cookies-mw: CookiesMiddleware diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst index b16066d0c..b3141cd12 100644 --- a/docs/topics/media-pipeline.rst +++ b/docs/topics/media-pipeline.rst @@ -564,6 +564,22 @@ See here the methods that you can override in your custom Files Pipeline: for file_url in adapter["file_urls"]: yield scrapy.Request(file_url) + This is also how you verify downloads against checksums published by the + website, using the :reqmeta:`expected_checksum` meta key: + + .. code-block:: python + + from scrapy import Request + + + def get_media_requests(self, item, info): + adapter = ItemAdapter(item) + for file_url, sha256 in zip(adapter["file_urls"], adapter["file_sha256"]): + yield Request(file_url, meta={"expected_checksum": {"sha256": sha256}}) + + Files whose checksum does not match are reported to + :meth:`~item_completed` as failures instead of being stored. + Those requests will be processed by the pipeline and, when they have finished downloading, the results will be sent to the :meth:`~item_completed` method, as a list of 2-element tuples. diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 75158440b..c1a370245 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -887,6 +887,7 @@ Those are: * :reqmeta:`download_slot` * :reqmeta:`download_warnsize` * :reqmeta:`download_timeout` +* :reqmeta:`expected_checksum` * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * ``ftp_user`` (See :setting:`FTP_USER` for more info) * :reqmeta:`give_up_log_level` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 65ee77258..b5bf669ba 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -896,6 +896,7 @@ Default: "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, + "scrapy.downloadermiddlewares.checksum.ChecksumMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, diff --git a/scrapy/downloadermiddlewares/checksum.py b/scrapy/downloadermiddlewares/checksum.py new file mode 100644 index 000000000..8417b6f26 --- /dev/null +++ b/scrapy/downloadermiddlewares/checksum.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING + +from scrapy.downloadermiddlewares.retry import get_retry_request +from scrapy.exceptions import ChecksumError + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + from scrapy.crawler import Crawler + from scrapy.http import Request, Response + + +class ChecksumMiddleware: + """Verifies response bodies against the checksums declared in the + :reqmeta:`expected_checksum` request meta key. + + .. versionadded:: VERSION""" + + def __init__(self, crawler: Crawler): + self.crawler: Crawler = crawler + + @classmethod + def from_crawler(cls, crawler: Crawler) -> Self: + return cls(crawler) + + def process_response( + self, request: Request, response: Response + ) -> Request | Response: + for algorithm, checksum in request.meta.get("expected_checksum", {}).items(): + expected = ( + bytes.fromhex(checksum) if isinstance(checksum, str) else checksum + ) + if hashlib.new(algorithm, response.body).digest() == expected: + continue + if not request.meta.get("dont_retry", False): + assert self.crawler.spider + new_request = get_retry_request( + request, + spider=self.crawler.spider, + reason=f"checksum/{algorithm}", + ) + if new_request: + return new_request + raise ChecksumError( + f"The {algorithm} checksum of the response body of {request} does " + f"not match the expected checksum." + ) + return response diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index ccaccddf2..a92cff30f 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -134,6 +134,15 @@ class UnsupportedURLSchemeError(Exception): """Indicates that the URL scheme is not supported.""" +class ChecksumError(Exception): + """Indicates that a response body does not match the checksum expected for + it. + + .. versionadded:: VERSION + + See :reqmeta:`expected_checksum`.""" + + # Items diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a44b36c8a..61df19db9 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -322,6 +322,7 @@ DOWNLOADER_MIDDLEWARES_BASE = { "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400, "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500, "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550, + "scrapy.downloadermiddlewares.checksum.ChecksumMiddleware": 560, "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580, "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590, "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600, diff --git a/tests/test_downloadermiddleware_checksum.py b/tests/test_downloadermiddleware_checksum.py new file mode 100644 index 000000000..9ed6bf554 --- /dev/null +++ b/tests/test_downloadermiddleware_checksum.py @@ -0,0 +1,143 @@ +from __future__ import annotations + +import hashlib +import logging +from typing import TYPE_CHECKING, Any + +import pytest +from itemadapter import ItemAdapter + +from scrapy import Request, Spider, signals +from scrapy.downloadermiddlewares.checksum import ChecksumMiddleware +from scrapy.exceptions import ChecksumError +from scrapy.http import Response +from scrapy.pipelines.files import FilesPipeline +from scrapy.utils.spider import DefaultSpider +from scrapy.utils.test import get_crawler +from tests.utils.decorators import coroutine_test + +if TYPE_CHECKING: + from collections.abc import AsyncIterator, Iterator + from pathlib import Path + + from scrapy.pipelines.media import MediaPipeline + from tests.mockserver.http import MockServer + + +BODY = b"file content to hash" +SHA256 = hashlib.sha256(BODY).hexdigest() +WRONG_SHA256 = "0" * 64 + + +class TestChecksumMiddleware: + def setup_method(self) -> None: + self.crawler = get_crawler(DefaultSpider, {"RETRY_TIMES": 1}) + self.crawler.spider = self.crawler._create_spider() + self.mw = ChecksumMiddleware.from_crawler(self.crawler) + + def _process(self, meta: dict[str, Any]) -> Request | Response: + return self.mw.process_response( + Request("https://example.com/file", meta=meta), + Response("https://example.com/file", body=BODY), + ) + + def test_no_expected_checksum(self) -> None: + assert isinstance(self._process({}), Response) + + @pytest.mark.parametrize( + "expected", [SHA256, SHA256.upper(), bytes.fromhex(SHA256)] + ) + def test_match(self, expected: str | bytes) -> None: + result = self._process({"expected_checksum": {"sha256": expected}}) + assert isinstance(result, Response) + + def test_mismatch_retries(self) -> None: + result = self._process({"expected_checksum": {"sha256": WRONG_SHA256}}) + assert isinstance(result, Request) + assert result.meta["retry_times"] == 1 + assert self.crawler.stats + assert self.crawler.stats.get_value("retry/reason_count/checksum/sha256") == 1 + + def test_mismatch_gives_up(self) -> None: + with pytest.raises(ChecksumError, match="sha256"): + self._process( + {"expected_checksum": {"sha256": WRONG_SHA256}, "retry_times": 1} + ) + + def test_dont_retry(self) -> None: + with pytest.raises(ChecksumError, match="sha256"): + self._process( + {"expected_checksum": {"sha256": WRONG_SHA256}, "dont_retry": True} + ) + + def test_every_algorithm_checked(self) -> None: + with pytest.raises(ChecksumError, match="sha256"): + self._process( + { + "expected_checksum": { + "sha512": hashlib.sha512(BODY).hexdigest(), + "sha256": WRONG_SHA256, + }, + "dont_retry": True, + } + ) + + +class ChecksumFilesPipeline(FilesPipeline): + def get_media_requests( + self, item: Any, info: MediaPipeline.SpiderInfo + ) -> list[Request]: + adapter = ItemAdapter(item) + return [ + Request(url, meta={"expected_checksum": {"sha256": checksum}}) + for url, checksum in zip( + adapter["file_urls"], adapter["file_sha256"], strict=True + ) + ] + + +class FileItemSpider(Spider): + name = "file_item" + + async def start(self) -> AsyncIterator[Request]: + yield Request(self.good_url) # type: ignore[attr-defined] + + def parse(self, response: Response) -> Iterator[Any]: + yield { + "file_urls": [self.good_url, self.bad_url], # type: ignore[attr-defined] + "file_sha256": [ + hashlib.sha256(b"Works").hexdigest(), + WRONG_SHA256, + ], + } + + +class TestMediaPipelineIntegration: + @coroutine_test + async def test_files_pipeline( + self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + items = [] + + def _on_item_scraped(item: Any) -> None: + items.append(item) + + crawler = get_crawler( + FileItemSpider, + { + "FILES_STORE": str(tmp_path), + "ITEM_PIPELINES": {ChecksumFilesPipeline: 1}, + "RETRY_TIMES": 0, + }, + ) + crawler.signals.connect(_on_item_scraped, signals.item_scraped) + with caplog.at_level(logging.WARNING): + await crawler.crawl_async( + good_url=mockserver.url("/text"), + bad_url=mockserver.url("/html"), + ) + + assert len(items) == 1 + assert [file["url"] for file in items[0]["files"]] == [mockserver.url("/text")] + assert "does not match the expected checksum" in caplog.text + assert len(list(tmp_path.glob("full/*"))) == 1