From 0c160882306a8e33f92815cc12dc8979f04f8f5e Mon Sep 17 00:00:00 2001
From: Eugenio Lacuesta <1731933+elacuesta@users.noreply.github.com>
Date: Thu, 11 Mar 2021 11:52:35 -0300
Subject: [PATCH] headers_received signal (#4897)

---
 docs/faq.rst                               | 11 +--
 docs/topics/exceptions.rst                 |  7 +-
 docs/topics/request-response.rst           |  6 +-
 docs/topics/signals.rst                    | 32 +++++++-
 scrapy/core/downloader/handlers/http11.py  | 24 ++++++
 scrapy/signals.py                          |  1 +
 tests/spiders.py                           | 29 ++++++++
 tests/test_crawl.py                        | 26 ++++++-
 tests/test_engine.py                       | 85 ++++++----------------
 tests/test_engine_stop_download_bytes.py   | 60 +++++++++++++++
 tests/test_engine_stop_download_headers.py | 56 ++++++++++++++
 11 files changed, 260 insertions(+), 77 deletions(-)
 create mode 100644 tests/test_engine_stop_download_bytes.py
 create mode 100644 tests/test_engine_stop_download_headers.py

diff --git a/docs/faq.rst b/docs/faq.rst
index f492dfa30..9709885f6 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -398,11 +398,12 @@ How can I cancel the download of a given response?
 --------------------------------------------------
 
 In some situations, it might be useful to stop the download of a certain response.
-For instance, if you only need the first part of a large response and you would like
-to save resources by avoiding the download of the whole body.
-In that case, you could attach a handler to the :class:`~scrapy.signals.bytes_received`
-signal and raise a :exc:`~scrapy.exceptions.StopDownload` exception. Please refer to
-the :ref:`topics-stop-response-download` topic for additional information and examples.
+For instance, sometimes you can determine whether or not you need the full contents
+of a response by inspecting its headers or the first bytes of its body. In that case,
+you could save resources by attaching a handler to the :class:`~scrapy.signals.bytes_received`
+or :class:`~scrapy.signals.headers_received` signals and raising a
+:exc:`~scrapy.exceptions.StopDownload` exception. Please refer to the
+:ref:`topics-stop-response-download` topic for additional information and examples.
 
 
 .. _has been reported: https://github.com/scrapy/scrapy/issues/2905
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 583a50ab8..2f1517906 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -85,8 +85,8 @@ StopDownload
 
 .. exception:: StopDownload(fail=True)
 
-Raised from a :class:`~scrapy.signals.bytes_received` signal handler to
-indicate that no further bytes should be downloaded for a response.
+Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
+signal handler to indicate that no further bytes should be downloaded for a response.
 
 The ``fail`` boolean parameter controls which method will handle the resulting
 response:
@@ -110,5 +110,6 @@ attribute.
     ``StopDownload(False)`` or ``StopDownload(True)`` will raise
     a :class:`TypeError`.
 
-See the documentation for the :class:`~scrapy.signals.bytes_received` signal
+See the documentation for the :class:`~scrapy.signals.bytes_received` and
+:class:`~scrapy.signals.headers_received` signals
 and the :ref:`topics-stop-response-download` topic for additional information and examples.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 98906992d..37008f3e9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -432,9 +432,9 @@ The meta key is used set retry times per request. When initialized, the
 Stopping the download of a Response
 ===================================
 
-Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a
-:class:`~scrapy.signals.bytes_received` signal handler will stop the
-download of a given response. See the following example::
+Raising a :exc:`~scrapy.exceptions.StopDownload` exception from a handler for the
+:class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
+signals will stop the download of a given response. See the following example::
 
     import scrapy
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 1d99d8c28..98cfa606c 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -384,6 +384,11 @@ bytes_received
     a possible scenario for a 25 kb response would be two signals fired
     with 10 kb of data, and a final one with 5 kb of data.
 
+    Handlers for this signal can stop the download of a response while it
+    is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
+    exception. Please refer to the :ref:`topics-stop-response-download` topic
+    for additional information and examples.
+
     This signal does not support returning deferreds from its handlers.
 
     :param data: the data received by the download handler
@@ -395,11 +400,36 @@ bytes_received
     :param spider: the spider associated with the response
     :type spider: :class:`~scrapy.spiders.Spider` object
 
-.. note:: Handlers of this signal can stop the download of a response while it
+headers_received
+~~~~~~~~~~~~~~~~
+
+.. versionadded:: VERSION
+
+.. signal:: headers_received
+.. function:: headers_received(headers, request, spider)
+
+    Sent by the HTTP 1.1 and S3 download handlers when the response headers are
+    available for a given request, before downloading any additional content.
+
+    Handlers for this signal can stop the download of a response while it
     is in progress by raising the :exc:`~scrapy.exceptions.StopDownload`
     exception. Please refer to the :ref:`topics-stop-response-download` topic
     for additional information and examples.
 
+    This signal does not support returning deferreds from its handlers.
+
+    :param headers: the headers received by the download handler
+    :type headers: :class:`scrapy.http.headers.Headers` object
+
+    :param body_length: expected size of the response body, in bytes
+    :type body_length: `int`
+
+    :param request: the request that generated the download
+    :type request: :class:`~scrapy.http.Request` object
+
+    :param spider: the spider associated with the response
+    :type spider: :class:`~scrapy.spiders.Spider` object
+
 Response signals
 ----------------
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 513df2de9..516a4326b 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -382,6 +382,29 @@ class ScrapyAgent:
         return result
 
     def _cb_bodyready(self, txresponse, request):
+        headers_received_result = self._crawler.signals.send_catch_log(
+            signal=signals.headers_received,
+            headers=Headers(txresponse.headers.getAllRawHeaders()),
+            body_length=txresponse.length,
+            request=request,
+            spider=self._crawler.spider,
+        )
+        for handler, result in headers_received_result:
+            if isinstance(result, Failure) and isinstance(result.value, StopDownload):
+                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
+                             {"request": request, "handler": handler.__qualname__})
+                txresponse._transport.stopProducing()
+                with suppress(AttributeError):
+                    txresponse._transport._producer.loseConnection()
+                return {
+                    "txresponse": txresponse,
+                    "body": b"",
+                    "flags": ["download_stopped"],
+                    "certificate": None,
+                    "ip_address": None,
+                    "failure": result if result.value.fail else None,
+                }
+
         # deliverBody hangs for responses without body
         if txresponse.length == 0:
             return {
@@ -529,6 +552,7 @@ class _ResponseReader(protocol.Protocol):
             if isinstance(result, Failure) and isinstance(result.value, StopDownload):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
+                self.transport.stopProducing()
                 self.transport._producer.loseConnection()
                 failure = result if result.value.fail else None
                 self._finish_response(flags=["download_stopped"], failure=failure)
diff --git a/scrapy/signals.py b/scrapy/signals.py
index c61ae6ec3..8cf2a4d93 100644
--- a/scrapy/signals.py
+++ b/scrapy/signals.py
@@ -17,6 +17,7 @@ request_reached_downloader = object()
 request_left_downloader = object()
 response_received = object()
 response_downloaded = object()
+headers_received = object()
 bytes_received = object()
 item_scraped = object()
 item_dropped = object()
diff --git a/tests/spiders.py b/tests/spiders.py
index 106392ea6..7e579098a 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -390,3 +390,32 @@ class BytesReceivedErrbackSpider(BytesReceivedCallbackSpider):
     def bytes_received(self, data, request, spider):
         self.meta["bytes_received"] = data
         raise StopDownload(fail=True)
+
+
+class HeadersReceivedCallbackSpider(MetaSpider):
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        spider = super().from_crawler(crawler, *args, **kwargs)
+        crawler.signals.connect(spider.headers_received, signals.headers_received)
+        return spider
+
+    def start_requests(self):
+        yield Request(self.mockserver.url("/status"), errback=self.errback)
+
+    def parse(self, response):
+        self.meta["response"] = response
+
+    def errback(self, failure):
+        self.meta["failure"] = failure
+
+    def headers_received(self, headers, body_length, request, spider):
+        self.meta["headers_received"] = headers
+        raise StopDownload(fail=False)
+
+
+class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
+
+    def headers_received(self, headers, body_length, request, spider):
+        self.meta["headers_received"] = headers
+        raise StopDownload(fail=True)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1083c1678..84bac9b50 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -35,6 +35,8 @@ from tests.spiders import (
     DelaySpider,
     DuplicateStartRequestsSpider,
     FollowAllSpider,
+    HeadersReceivedCallbackSpider,
+    HeadersReceivedErrbackSpider,
     SimpleSpider,
     SingleRequestSpider,
 )
@@ -496,7 +498,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
 
     @defer.inlineCallbacks
-    def test_stop_download_callback(self):
+    def test_bytes_received_stop_download_callback(self):
         crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
@@ -505,7 +507,7 @@ class CrawlSpiderTestCase(TestCase):
         self.assertLess(len(crawler.spider.meta["response"].body), crawler.spider.full_response_length)
 
     @defer.inlineCallbacks
-    def test_stop_download_errback(self):
+    def test_bytes_received_stop_download_errback(self):
         crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
@@ -518,3 +520,23 @@ class CrawlSpiderTestCase(TestCase):
         self.assertLess(
             len(crawler.spider.meta["failure"].value.response.body),
             crawler.spider.full_response_length)
+
+    @defer.inlineCallbacks
+    def test_headers_received_stop_download_callback(self):
+        crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("failure"))
+        self.assertIsInstance(crawler.spider.meta["response"], Response)
+        self.assertEqual(crawler.spider.meta["response"].headers, crawler.spider.meta.get("headers_received"))
+
+    @defer.inlineCallbacks
+    def test_headers_received_stop_download_errback(self):
+        crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+        yield crawler.crawl(mockserver=self.mockserver)
+        self.assertIsNone(crawler.spider.meta.get("response"))
+        self.assertIsInstance(crawler.spider.meta["failure"], Failure)
+        self.assertIsInstance(crawler.spider.meta["failure"].value, StopDownload)
+        self.assertIsInstance(crawler.spider.meta["failure"].value.response, Response)
+        self.assertEqual(
+            crawler.spider.meta["failure"].value.response.headers,
+            crawler.spider.meta.get("headers_received"))
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 3629aa1aa..ef1204f94 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -19,14 +19,12 @@ from urllib.parse import urlparse
 import attr
 from itemadapter import ItemAdapter
 from pydispatch import dispatcher
-from testfixtures import LogCapture
 from twisted.internet import defer, reactor
 from twisted.trial import unittest
 from twisted.web import server, static, util
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.item import Item, Field
 from scrapy.linkextractors import LinkExtractor
@@ -143,6 +141,7 @@ class CrawlerRun:
         self.reqreached = []
         self.itemerror = []
         self.itemresp = []
+        self.headers = {}
         self.bytes = defaultdict(lambda: list())
         self.signals_caught = {}
         self.spider_class = spider_class
@@ -165,6 +164,7 @@ class CrawlerRun:
         self.crawler = get_crawler(self.spider_class)
         self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
         self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.headers_received, signals.headers_received)
         self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
         self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
         self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
@@ -183,6 +183,7 @@ class CrawlerRun:
             if not name.startswith('_'):
                 disconnect_all(signal)
         self.deferred.callback(None)
+        return self.crawler.stop()
 
     def geturl(self, path):
         return f"http://localhost:{self.portno}{path}"
@@ -197,6 +198,9 @@ class CrawlerRun:
     def item_scraped(self, item, spider, response):
         self.itemresp.append((item, response))
 
+    def headers_received(self, headers, body_length, request, spider):
+        self.headers[request] = headers
+
     def bytes_received(self, data, request, spider):
         self.bytes[request].append(data)
 
@@ -220,18 +224,7 @@ class CrawlerRun:
         self.signals_caught[sig] = signalargs
 
 
-class StopDownloadCrawlerRun(CrawlerRun):
-    """
-    Make sure raising the StopDownload exception stops the download of the response body
-    """
-
-    def bytes_received(self, data, request, spider):
-        super().bytes_received(data, request, spider)
-        raise StopDownload(fail=False)
-
-
 class EngineTest(unittest.TestCase):
-
     @defer.inlineCallbacks
     def test_crawler(self):
 
@@ -241,8 +234,8 @@ class EngineTest(unittest.TestCase):
             self.run = CrawlerRun(spider)
             yield self.run.run()
             self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=9)
-            self._assert_downloaded_responses()
+            self._assert_scheduled_requests(count=9)
+            self._assert_downloaded_responses(count=9)
             self._assert_scraped_items()
             self._assert_signals_caught()
             self._assert_bytes_received()
@@ -251,7 +244,7 @@ class EngineTest(unittest.TestCase):
     def test_crawler_dupefilter(self):
         self.run = CrawlerRun(TestDupeFilterSpider)
         yield self.run.run()
-        self._assert_scheduled_requests(urls_to_visit=8)
+        self._assert_scheduled_requests(count=8)
         self._assert_dropped_requests()
 
     @defer.inlineCallbacks
@@ -267,8 +260,8 @@ class EngineTest(unittest.TestCase):
         urls_expected = {self.run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
 
-    def _assert_scheduled_requests(self, urls_to_visit=None):
-        self.assertEqual(urls_to_visit, len(self.run.reqplug))
+    def _assert_scheduled_requests(self, count=None):
+        self.assertEqual(count, len(self.run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
@@ -286,10 +279,10 @@ class EngineTest(unittest.TestCase):
     def _assert_dropped_requests(self):
         self.assertEqual(len(self.run.reqdropped), 1)
 
-    def _assert_downloaded_responses(self):
+    def _assert_downloaded_responses(self, count):
         # response tests
-        self.assertEqual(9, len(self.run.respplug))
-        self.assertEqual(9, len(self.run.reqreached))
+        self.assertEqual(count, len(self.run.respplug))
+        self.assertEqual(count, len(self.run.reqreached))
 
         for response, _ in self.run.respplug:
             if self.run.getpath(response.url) == '/item999.html':
@@ -323,6 +316,13 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
+    def _assert_headers_received(self):
+        for headers in self.run.headers.values():
+            self.assertIn(b"Server", headers)
+            self.assertIn(b"TwistedWeb", headers[b"Server"])
+            self.assertIn(b"Date", headers)
+            self.assertIn(b"Content-Type", headers)
+
     def _assert_bytes_received(self):
         self.assertEqual(9, len(self.run.bytes))
         for request, data in self.run.bytes.items():
@@ -371,6 +371,7 @@ class EngineTest(unittest.TestCase):
         assert signals.spider_opened in self.run.signals_caught
         assert signals.spider_idle in self.run.signals_caught
         assert signals.spider_closed in self.run.signals_caught
+        assert signals.headers_received in self.run.signals_caught
 
         self.assertEqual({'spider': self.run.spider},
                          self.run.signals_caught[signals.spider_opened])
@@ -403,48 +404,6 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(len(e.open_spiders), 0)
 
 
-class StopDownloadEngineTest(EngineTest):
-
-    @defer.inlineCallbacks
-    def test_crawler(self):
-        for spider in TestSpider, DictItemsSpider:
-            self.run = StopDownloadCrawlerRun(spider)
-            with LogCapture() as log:
-                yield self.run.run()
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-                log.check_present(("scrapy.core.downloader.handlers.http11",
-                                   "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
-                                   "from signal handler"
-                                   " StopDownloadCrawlerRun.bytes_received"))
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(urls_to_visit=9)
-            self._assert_downloaded_responses()
-            self._assert_signals_caught()
-            self._assert_bytes_received()
-
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
-            joined_data = b"".join(data)
-            self.assertTrue(len(data) == 1)  # signal was fired only once
-            if self.run.getpath(request.url) == "/numbers":
-                # Received bytes are not the complete response. The exact amount depends
-                # on the buffer size, which can vary, so we only check that the amount
-                # of received bytes is strictly less than the full response.
-                numbers = [str(x).encode("utf8") for x in range(2**18)]
-                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
-
-
 if __name__ == "__main__":
     if len(sys.argv) > 1 and sys.argv[1] == 'runserver':
         start_test_site(debug=True)
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
new file mode 100644
index 000000000..0ba69e096
--- /dev/null
+++ b/tests/test_engine_stop_download_bytes.py
@@ -0,0 +1,60 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+
+from scrapy.exceptions import StopDownload
+
+from tests.test_engine import (
+    AttrsItemsSpider,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    TestSpider,
+    CrawlerRun,
+    EngineTest,
+)
+
+
+class BytesReceivedCrawlerRun(CrawlerRun):
+    def bytes_received(self, data, request, spider):
+        super().bytes_received(data, request, spider)
+        raise StopDownload(fail=False)
+
+
+class BytesReceivedEngineTest(EngineTest):
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
+            self.run = BytesReceivedCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
+                                   "from signal handler BytesReceivedCrawlerRun.bytes_received"))
+            self._assert_visited_urls()
+            self._assert_scheduled_requests(count=9)
+            self._assert_downloaded_responses(count=9)
+            self._assert_signals_caught()
+            self._assert_headers_received()
+            self._assert_bytes_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(9, len(self.run.bytes))
+        for request, data in self.run.bytes.items():
+            joined_data = b"".join(data)
+            self.assertTrue(len(data) == 1)  # signal was fired only once
+            if self.run.getpath(request.url) == "/numbers":
+                # Received bytes are not the complete response. The exact amount depends
+                # on the buffer size, which can vary, so we only check that the amount
+                # of received bytes is strictly less than the full response.
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                self.assertTrue(len(joined_data) < len(b"".join(numbers)))
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
new file mode 100644
index 000000000..fad6643ad
--- /dev/null
+++ b/tests/test_engine_stop_download_headers.py
@@ -0,0 +1,56 @@
+from testfixtures import LogCapture
+from twisted.internet import defer
+
+from scrapy.exceptions import StopDownload
+
+from tests.test_engine import (
+    AttrsItemsSpider,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    TestSpider,
+    CrawlerRun,
+    EngineTest,
+)
+
+
+class HeadersReceivedCrawlerRun(CrawlerRun):
+    def headers_received(self, headers, body_length, request, spider):
+        super().headers_received(headers, body_length, request, spider)
+        raise StopDownload(fail=False)
+
+
+class HeadersReceivedEngineTest(EngineTest):
+    @defer.inlineCallbacks
+    def test_crawler(self):
+        for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
+            if spider is None:
+                continue
+            self.run = HeadersReceivedCrawlerRun(spider)
+            with LogCapture() as log:
+                yield self.run.run()
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
+                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
+                                   " handler HeadersReceivedCrawlerRun.headers_received"))
+                log.check_present(("scrapy.core.downloader.handlers.http11",
+                                   "DEBUG",
+                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
+                                   " signal handler HeadersReceivedCrawlerRun.headers_received"))
+            self._assert_visited_urls()
+            self._assert_downloaded_responses(count=6)
+            self._assert_signals_caught()
+            self._assert_bytes_received()
+            self._assert_headers_received()
+
+    def _assert_bytes_received(self):
+        self.assertEqual(0, len(self.run.bytes))
+
+    def _assert_visited_urls(self):
+        must_be_visited = ["/", "/redirect", "/redirected"]
+        urls_visited = {rp[0].url for rp in self.run.respplug}
+        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"