Notify spider when requests are ignored by HttpErrorMiddleware, and generally when any call to process_spider_input raises an exception

docs/topics/spider-middleware.rst

@@ -64,19 +64,18 @@ single Python class that defines one or more of the following methods:
         This method is called for each response that goes through the spider
         middleware and into the spider, for processing.
 
-        :meth:`process_spider_input` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item`
-        objects.
+        :meth:`process_spider_input` should return ``None`` or raise and
+        exception.
 
-        If returns ``None``, Scrapy will continue processing this response,
+        If it returns ``None``, Scrapy will continue processing this response,
         executing all other middlewares until, finally, the response is handled
         to the spider for processing.
 
-        If returns an iterable, Scrapy won't bother calling any other spider
-        middleware :meth:`process_spider_input` and will return the iterable
-        back in the other direction for the :meth:`process_spider_output` to
-        process it, or the :meth:`process_spider_exception` if it raised an
-        exception.
+        If it raises an exception, Scrapy won't bother calling any other spider
+        middleware :meth:`process_spider_input` and will call the request
+        errback.  The output of the errback is chained back in the other
+        direction for :meth:`process_spider_output` to process it, or
+        :meth:`process_spider_exception` if it raised an exception.
 
         :param reponse: the response being processed
         :type response: :class:`~scrapy.http.Response` object
@@ -89,7 +88,7 @@ single Python class that defines one or more of the following methods:
 
         This method is called with the results returned from the Spider, after
         it has processed the response.
-     
+
         :meth:`process_spider_output` must return an iterable of
         :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` objects.
 

scrapy/contrib/spidermiddleware/httperror.py

@@ -3,6 +3,16 @@ HttpError Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+from scrapy.core.exceptions import IgnoreRequest
+
+
+class HttpErrorException(IgnoreRequest):
+    """A non-200 response was filtered"""
+
+    def __init__(self, response, *args, **kwargs):
+        self.response = response
+        super(HttpErrorException, self).__init__(*args, **kwargs)
+
 
 class HttpErrorMiddleware(object):
 
@@ -15,4 +25,5 @@ class HttpErrorMiddleware(object):
             allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
         if response.status in allowed_statuses:
             return
-        return []
+        raise HttpErrorException(response, 'Ignoring non-200 response')
+

scrapy/spider/middleware.py

@@ -7,11 +7,11 @@ docs/topics/spider-middleware.rst
 """
 
 from scrapy import log
+from twisted.python.failure import Failure
 from scrapy.core.exceptions import NotConfigured
 from scrapy.utils.misc import load_object
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
-from scrapy.http import Request
 from scrapy.conf import settings
 
 def _isiterable(possible_iterator):
@@ -60,12 +60,14 @@ class SpiderMiddlewareManager(object):
 
         def process_spider_input(response):
             for method in self.spider_middleware:
-                result = method(response=response, spider=spider)
-                assert result is None or _isiterable(result), \
-                    'Middleware %s must returns None or an iterable object, got %s ' % \
-                    (fname(method), type(result))
-                if result is not None:
-                    return result
+                try:
+                    result = method(response=response, spider=spider)
+                    assert result is None, \
+                            'Middleware %s must returns None or ' \
+                            'raise an exception, got %s ' \
+                            % (fname(method), type(result))
+                except:
+                    return scrape_func(Failure(), request, spider)
             return scrape_func(response, request, spider)
 
         def process_spider_exception(_failure):

scrapy/tests/test_spidermiddleware_httperror.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
-from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware
+from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException
 
 
 class TestHttpErrorMiddleware(TestCase):
@@ -18,21 +18,19 @@ class TestHttpErrorMiddleware(TestCase):
         self.res404.request = self.req
 
     def test_process_spider_input(self):
-        self.assertEquals(self.mw.process_spider_input(self.res200, self.spider),
-                          None)
-
-        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
-                          [])
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpErrorException,
+                self.mw.process_spider_input, self.res404, self.spider)
 
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',
                               meta={'handle_httpstatus_list': [404]})
-
-        self.assertEquals(self.mw.process_spider_input(res, self.spider),
-                          None)
+        self.assertEquals(None,
+            self.mw.process_spider_input(res, self.spider))
 
         self.spider.handle_httpstatus_list = [404]
-        self.assertEquals(self.mw.process_spider_input(self.res404, self.spider),
-                          None)
+        self.assertEquals(None,
+            self.mw.process_spider_input(self.res404, self.spider))