Merge pull request #6766 from wRAR/errback-tests

Improve test coverage of Scraper.

scrapy/core/scraper.py

@@ -374,32 +374,24 @@ class Scraper:
         errors that got propagated thru here).
 
         spider_failure: the value passed into the errback of self.call_spider()
+        (likely raised in the request errback)
+
         download_failure: the value passed into _scrape2() from
         ExecutionEngine._handle_downloader_output() as "result"
+        (likely raised in the download handler or a downloader middleware)
         """
         if not download_failure.check(IgnoreRequest):
             assert self.crawler.spider
-            if download_failure.frames:
-                logkws = self.logformatter.download_error(
-                    download_failure, request, self.crawler.spider
-                )
-                logger.log(
-                    *logformatter_adapter(logkws),
-                    extra={"spider": self.crawler.spider},
-                    exc_info=failure_to_exc_info(download_failure),
-                )
-            else:
-                errmsg = download_failure.getErrorMessage()
-                if errmsg:
-                    logkws = self.logformatter.download_error(
-                        download_failure, request, self.crawler.spider, errmsg
-                    )
-                    logger.log(
-                        *logformatter_adapter(logkws),
-                        extra={"spider": self.crawler.spider},
-                    )
-
+            logkws = self.logformatter.download_error(
+                download_failure, request, self.crawler.spider
+            )
+            logger.log(
+                *logformatter_adapter(logkws),
+                extra={"spider": self.crawler.spider},
+                exc_info=failure_to_exc_info(download_failure),
+            )
         if spider_failure is not download_failure:
+            # a request errback raised a different exception, it needs to be handled later
             return spider_failure
         return None
 

tests/test_crawl.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import logging
 import unittest
@@ -14,7 +16,7 @@ from twisted.trial.unittest import TestCase
 
 from scrapy import signals
 from scrapy.crawler import CrawlerRunner
-from scrapy.exceptions import StopDownload
+from scrapy.exceptions import CloseSpider, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
@@ -723,3 +725,100 @@ class TestCrawlSpider(TestCase):
         assert crawler.spider.meta[
             "failure"
         ].value.response.headers == crawler.spider.meta.get("headers_received")
+
+    @defer.inlineCallbacks
+    def test_spider_errback(self):
+        failures = []
+
+        def eb(failure: Failure) -> Failure:
+            failures.append(failure)
+            return failure
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert len(failures) == 1
+        assert "HTTP status code is not handled or not allowed" in str(log)
+        assert "Spider error processing" not in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_silence(self):
+        failures = []
+
+        def eb(failure: Failure) -> None:
+            failures.append(failure)
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert len(failures) == 1
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_exception(self):
+        def eb(failure: Failure) -> None:
+            raise ValueError("foo")
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert "Spider error processing" in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_downloader_error(self):
+        failures = []
+
+        def eb(failure: Failure) -> Failure:
+            failures.append(failure)
+            return failure
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert len(failures) == 1
+        assert "Error downloading" in str(log)
+        assert "Spider error processing" not in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_exception_downloader_error(self):
+        def eb(failure: Failure) -> None:
+            raise ValueError("foo")
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert "Error downloading" in str(log)
+        assert "Spider error processing" in str(log)
+
+    @defer.inlineCallbacks
+    def test_raise_closespider(self):
+        def cb(response):
+            raise CloseSpider
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
+        assert "Closing spider (cancelled)" in str(log)
+        assert "Spider error processing" not in str(log)
+
+    @defer.inlineCallbacks
+    def test_raise_closespider_reason(self):
+        def cb(response):
+            raise CloseSpider("my_reason")
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
+        assert "Closing spider (my_reason)" in str(log)
+        assert "Spider error processing" not in str(log)