Update docs, add tests, remove FIXME comment

docs/topics/spider-middleware.rst

@@ -78,7 +78,8 @@ following methods:
 
         If it raises an exception, Scrapy won't bother calling any other spider
         middleware :meth:`process_spider_input` and will call the request
-        errback.  The output of the errback is chained back in the other
+        errback if there is one, otherwise it will start the :meth:`process_spider_exception`
+        chain. The output of the errback is chained back in the other
         direction for :meth:`process_spider_output` to process it, or
         :meth:`process_spider_exception` if it raised an exception.
 

scrapy/core/scraper.py

@@ -135,7 +135,6 @@ class Scraper(object):
             return self.spidermw.scrape_response(
                 self.call_spider, request_result, request, spider)
         else:
-            # FIXME: don't ignore errors in spider middleware
             dfd = self.call_spider(request_result, request, spider)
             return dfd.addErrback(
                 self._log_download_errors, request_result, request, spider)

tests/test_spidermiddleware.py

@@ -87,7 +87,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
 
     def test_process_spider_exception_return_none(self):
 
-        class ProcessSpiderOutputExceptionReturnNoneMiddleware:
+        class ProcessSpiderExceptionReturnNoneMiddleware:
             def process_spider_exception(self, response, exception, spider):
                 return None
 
@@ -95,7 +95,7 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
             def process_spider_output(self, response, result, spider):
                 1/0
 
-        self.mwman._add_middleware(ProcessSpiderOutputExceptionReturnNoneMiddleware())
+        self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)

tests/test_spidermiddleware_output_chain.py

@@ -45,8 +45,13 @@ class RecoveryMiddleware:
 
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
-class ProcessSpiderInputSpider(Spider):
-    name = 'ProcessSpiderInputSpider'
+class FailProcessSpiderInputMiddleware:
+    def process_spider_input(self, response, spider):
+        spider.logger.info('Middleware: will raise IndexError')
+        raise IndexError()
+
+class ProcessSpiderInputSpiderWithoutErrback(Spider):
+    name = 'ProcessSpiderInputSpiderWithoutErrback'
     custom_settings = {
         'SPIDER_MIDDLEWARES': {
             # spider
@@ -58,23 +63,23 @@ class ProcessSpiderInputSpider(Spider):
     }
 
     def start_requests(self):
-        yield Request(url=self.mockserver.url('/status?n=200'),
-                      callback=self.parse, errback=self.errback)
+        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse)
 
     def parse(self, response):
         return {'from': 'callback'}
 
+
+class ProcessSpiderInputSpiderWithErrback(ProcessSpiderInputSpiderWithoutErrback):
+    name = 'ProcessSpiderInputSpiderWithErrback'
+
+    def start_requests(self):
+        yield Request(url=self.mockserver.url('/status?n=200'), callback=self.parse, errback=self.errback)
+
     def errback(self, failure):
         self.logger.info('Got a Failure on the Request errback')
         return {'from': 'errback'}
 
 
-class FailProcessSpiderInputMiddleware:
-    def process_spider_input(self, response, spider):
-        spider.logger.info('Middleware: will raise IndexError')
-        raise IndexError()
-
-
 # ================================================================================
 # (2) exceptions from a spider callback (generator)
 class GeneratorCallbackSpider(Spider):
@@ -278,12 +283,22 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("'item_scraped_count': 3", str(log))
 
     @defer.inlineCallbacks
-    def test_process_spider_input_errback(self):
+    def test_process_spider_input_without_errback(self):
         """
-        (1) An exception from the process_spider_input chain should not be caught by the
-        process_spider_exception chain, it should go directly to the Request errback
+        (1.1) An exception from the process_spider_input chain should be caught by the
+        process_spider_exception chain from the start if the Request has no errback
         """
-        log1 = yield self.crawl_log(ProcessSpiderInputSpider)
+        log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
+        self.assertIn("Middleware: will raise IndexError", str(log1))
+        self.assertIn("Middleware: IndexError exception caught", str(log1))
+
+    @defer.inlineCallbacks
+    def test_process_spider_input_with_errback(self):
+        """
+        (1.2) An exception from the process_spider_input chain should not be caught by the
+        process_spider_exception chain if the Request has an errback
+        """
+        log1 = yield self.crawl_log(ProcessSpiderInputSpiderWithErrback)
         self.assertNotIn("Middleware: IndexError exception caught", str(log1))
         self.assertIn("Middleware: will raise IndexError", str(log1))
         self.assertIn("Got a Failure on the Request errback", str(log1))