Automated merge with ssh://hg.scrapy.org/scrapy

scrapy/contrib/spidermiddleware/httperror.py

@@ -6,12 +6,12 @@ See documentation in docs/topics/spider-middleware.rst
 from scrapy.core.exceptions import IgnoreRequest
 
 
-class HttpErrorException(IgnoreRequest):
+class HttpError(IgnoreRequest):
     """A non-200 response was filtered"""
 
     def __init__(self, response, *args, **kwargs):
         self.response = response
-        super(HttpErrorException, self).__init__(*args, **kwargs)
+        super(HttpError, self).__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware(object):
@@ -25,5 +25,8 @@ class HttpErrorMiddleware(object):
             allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
         if response.status in allowed_statuses:
             return
-        raise HttpErrorException(response, 'Ignoring non-200 response')
+        raise HttpError(response, 'Ignoring non-200 response')
 
+    def process_spider_exception(self, response, exception, spider):
+        if isinstance(exception, HttpError):
+            return []

scrapy/tests/test_spidermiddleware_httperror.py

@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
-from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException
+from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError
 
 
 class TestHttpErrorMiddleware(TestCase):
@@ -20,9 +20,17 @@ class TestHttpErrorMiddleware(TestCase):
     def test_process_spider_input(self):
         self.assertEquals(None,
                 self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpErrorException,
+        self.assertRaises(HttpError,
                 self.mw.process_spider_input, self.res404, self.spider)
 
+    def test_process_spider_exception(self):
+        self.assertEquals([],
+                self.mw.process_spider_exception(self.res404, \
+                        HttpError(self.res404), self.spider))
+        self.assertEquals(None,
+                self.mw.process_spider_exception(self.res404, \
+                        Exception(), self.spider))
+
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',