Merge 448a432387 into e28e56aa61

docs/topics/downloader-middleware.rst

@@ -224,6 +224,42 @@ middleware, see the :ref:`downloader middleware usage guide
 For a list of the components enabled by default (and their orders) see the
 :setting:`DOWNLOADER_MIDDLEWARES_BASE` setting.
 
+ChecksumMiddleware
+------------------
+
+.. autoclass:: scrapy.downloadermiddlewares.checksum.ChecksumMiddleware
+
+.. autoexception:: scrapy.exceptions.ChecksumError
+
+.. reqmeta:: expected_checksum
+
+expected_checksum
+~~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+A dict that maps :mod:`hashlib` algorithm names to the expected checksum of the
+response body, as :class:`bytes` or as a hexadecimal :class:`str`:
+
+.. invisible-code-block: python
+
+    from scrapy import Request
+
+.. code-block:: python
+
+    Request(
+        "https://example.com/product1.pdf",
+        meta={
+            "expected_checksum": {
+                "sha256": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"
+            }
+        },
+    )
+
+A mismatch is retried like any other download failure, and once retries are
+exhausted the request fails with :exc:`~scrapy.exceptions.ChecksumError`. Set
+the :reqmeta:`dont_retry` meta key to fail on the first mismatch.
+
 CookiesMiddleware
 -----------------
 

docs/topics/media-pipeline.rst

@@ -595,6 +595,22 @@ See here the methods that you can override in your custom Files Pipeline:
              for file_url in adapter["file_urls"]:
                  yield scrapy.Request(file_url)
 
+      This is also how you verify downloads against checksums published by the
+      website, using the :reqmeta:`expected_checksum` meta key:
+
+      .. code-block:: python
+
+         from scrapy import Request
+
+
+         def get_media_requests(self, item, info):
+             adapter = ItemAdapter(item)
+             for file_url, sha256 in zip(adapter["file_urls"], adapter["file_sha256"]):
+                 yield Request(file_url, meta={"expected_checksum": {"sha256": sha256}})
+
+      Files whose checksum does not match are reported to
+      :meth:`~item_completed` as failures instead of being stored.
+
       Those requests will be processed by the pipeline and, when they have finished
       downloading, the results will be sent to the
       :meth:`~item_completed` method, as a list of 2-element tuples.

docs/topics/request-response.rst

@@ -839,6 +839,7 @@ Those are:
 * :reqmeta:`download_slot`
 * :reqmeta:`download_warnsize`
 * :reqmeta:`download_timeout`
+* :reqmeta:`expected_checksum`
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
 * ``ftp_user`` (See :setting:`FTP_USER` for more info)
 * :reqmeta:`give_up_log_level`

docs/topics/settings.rst

@@ -915,6 +915,7 @@ Default:
         "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
         "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
         "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+        "scrapy.downloadermiddlewares.checksum.ChecksumMiddleware": 560,
         "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
         "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
         "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,

scrapy/downloadermiddlewares/checksum.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import hashlib
+from typing import TYPE_CHECKING
+
+from scrapy.downloadermiddlewares.retry import get_retry_request
+from scrapy.exceptions import ChecksumError
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+    from scrapy.crawler import Crawler
+    from scrapy.http import Request, Response
+
+
+class ChecksumMiddleware:
+    """Verifies response bodies against the checksums declared in the
+    :reqmeta:`expected_checksum` request meta key.
+
+    .. versionadded:: VERSION"""
+
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        return cls(crawler)
+
+    def process_response(
+        self, request: Request, response: Response
+    ) -> Request | Response:
+        for algorithm, checksum in request.meta.get("expected_checksum", {}).items():
+            expected = (
+                bytes.fromhex(checksum) if isinstance(checksum, str) else checksum
+            )
+            if hashlib.new(algorithm, response.body).digest() == expected:
+                continue
+            if not request.meta.get("dont_retry", False):
+                assert self.crawler.spider
+                new_request = get_retry_request(
+                    request,
+                    spider=self.crawler.spider,
+                    reason=f"checksum/{algorithm}",
+                )
+                if new_request:
+                    return new_request
+            raise ChecksumError(
+                f"The {algorithm} checksum of the response body of {request} does "
+                f"not match the expected checksum."
+            )
+        return response

scrapy/exceptions.py

@@ -137,6 +137,15 @@ class UnsupportedURLSchemeError(Exception):
     """Indicates that the URL scheme is not supported."""
 
 
+class ChecksumError(Exception):
+    """Indicates that a response body does not match the checksum expected for
+    it.
+
+    .. versionadded:: VERSION
+
+    See :reqmeta:`expected_checksum`."""
+
+
 # Items
 
 

scrapy/settings/default_settings.py

@@ -323,6 +323,7 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
     "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
     "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
+    "scrapy.downloadermiddlewares.checksum.ChecksumMiddleware": 560,
     "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
     "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
     "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,

tests/test_downloadermiddleware_checksum.py

@@ -0,0 +1,143 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from typing import TYPE_CHECKING, Any
+
+import pytest
+from itemadapter import ItemAdapter
+
+from scrapy import Request, Spider, signals
+from scrapy.downloadermiddlewares.checksum import ChecksumMiddleware
+from scrapy.exceptions import ChecksumError
+from scrapy.http import Response
+from scrapy.pipelines.files import FilesPipeline
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Iterator
+    from pathlib import Path
+
+    from scrapy.pipelines.media import MediaPipeline
+    from tests.mockserver.http import MockServer
+
+
+BODY = b"file content to hash"
+SHA256 = hashlib.sha256(BODY).hexdigest()
+WRONG_SHA256 = "0" * 64
+
+
+class TestChecksumMiddleware:
+    def setup_method(self) -> None:
+        self.crawler = get_crawler(DefaultSpider, {"RETRY_TIMES": 1})
+        self.crawler.spider = self.crawler._create_spider()
+        self.mw = ChecksumMiddleware.from_crawler(self.crawler)
+
+    def _process(self, meta: dict[str, Any]) -> Request | Response:
+        return self.mw.process_response(
+            Request("https://example.com/file", meta=meta),
+            Response("https://example.com/file", body=BODY),
+        )
+
+    def test_no_expected_checksum(self) -> None:
+        assert isinstance(self._process({}), Response)
+
+    @pytest.mark.parametrize(
+        "expected", [SHA256, SHA256.upper(), bytes.fromhex(SHA256)]
+    )
+    def test_match(self, expected: str | bytes) -> None:
+        result = self._process({"expected_checksum": {"sha256": expected}})
+        assert isinstance(result, Response)
+
+    def test_mismatch_retries(self) -> None:
+        result = self._process({"expected_checksum": {"sha256": WRONG_SHA256}})
+        assert isinstance(result, Request)
+        assert result.meta["retry_times"] == 1
+        assert self.crawler.stats
+        assert self.crawler.stats.get_value("retry/reason_count/checksum/sha256") == 1
+
+    def test_mismatch_gives_up(self) -> None:
+        with pytest.raises(ChecksumError, match="sha256"):
+            self._process(
+                {"expected_checksum": {"sha256": WRONG_SHA256}, "retry_times": 1}
+            )
+
+    def test_dont_retry(self) -> None:
+        with pytest.raises(ChecksumError, match="sha256"):
+            self._process(
+                {"expected_checksum": {"sha256": WRONG_SHA256}, "dont_retry": True}
+            )
+
+    def test_every_algorithm_checked(self) -> None:
+        with pytest.raises(ChecksumError, match="sha256"):
+            self._process(
+                {
+                    "expected_checksum": {
+                        "sha512": hashlib.sha512(BODY).hexdigest(),
+                        "sha256": WRONG_SHA256,
+                    },
+                    "dont_retry": True,
+                }
+            )
+
+
+class ChecksumFilesPipeline(FilesPipeline):
+    def get_media_requests(
+        self, item: Any, info: MediaPipeline.SpiderInfo
+    ) -> list[Request]:
+        adapter = ItemAdapter(item)
+        return [
+            Request(url, meta={"expected_checksum": {"sha256": checksum}})
+            for url, checksum in zip(
+                adapter["file_urls"], adapter["file_sha256"], strict=True
+            )
+        ]
+
+
+class FileItemSpider(Spider):
+    name = "file_item"
+
+    async def start(self) -> AsyncIterator[Request]:
+        yield Request(self.good_url)  # type: ignore[attr-defined]
+
+    def parse(self, response: Response) -> Iterator[Any]:
+        yield {
+            "file_urls": [self.good_url, self.bad_url],  # type: ignore[attr-defined]
+            "file_sha256": [
+                hashlib.sha256(b"Works").hexdigest(),
+                WRONG_SHA256,
+            ],
+        }
+
+
+class TestMediaPipelineIntegration:
+    @coroutine_test
+    async def test_files_pipeline(
+        self, mockserver: MockServer, tmp_path: Path, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        items = []
+
+        def _on_item_scraped(item: Any) -> None:
+            items.append(item)
+
+        crawler = get_crawler(
+            FileItemSpider,
+            {
+                "FILES_STORE": str(tmp_path),
+                "ITEM_PIPELINES": {ChecksumFilesPipeline: 1},
+                "RETRY_TIMES": 0,
+            },
+        )
+        crawler.signals.connect(_on_item_scraped, signals.item_scraped)
+        with caplog.at_level(logging.WARNING):
+            await crawler.crawl_async(
+                good_url=mockserver.url("/text"),
+                bad_url=mockserver.url("/html"),
+            )
+
+        assert len(items) == 1
+        assert [file["url"] for file in items[0]["files"]] == [mockserver.url("/text")]
+        assert "does not match the expected checksum" in caplog.text
+        assert len(list(tmp_path.glob("full/*"))) == 1