From cce9c4da492f1a383c5cb9c80f7e07495267021e Mon Sep 17 00:00:00 2001
From: Daniel Grana <dangra@gmail.com>
Date: Fri, 21 May 2010 17:54:32 -0300
Subject: [PATCH] silence HttpError exceptions raised by httperror
 spidermiddleware if not handled by spider

---
 scrapy/contrib/spidermiddleware/httperror.py    |  9 ++++++---
 scrapy/tests/test_spidermiddleware_httperror.py | 12 ++++++++++--
 2 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
index 6da26aa2d..b76b7494b 100644
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ b/scrapy/contrib/spidermiddleware/httperror.py
@@ -6,12 +6,12 @@ See documentation in docs/topics/spider-middleware.rst
 from scrapy.core.exceptions import IgnoreRequest
 
 
-class HttpErrorException(IgnoreRequest):
+class HttpError(IgnoreRequest):
     """A non-200 response was filtered"""
 
     def __init__(self, response, *args, **kwargs):
         self.response = response
-        super(HttpErrorException, self).__init__(*args, **kwargs)
+        super(HttpError, self).__init__(*args, **kwargs)
 
 
 class HttpErrorMiddleware(object):
@@ -25,5 +25,8 @@ class HttpErrorMiddleware(object):
             allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
         if response.status in allowed_statuses:
             return
-        raise HttpErrorException(response, 'Ignoring non-200 response')
+        raise HttpError(response, 'Ignoring non-200 response')
 
+    def process_spider_exception(self, response, exception, spider):
+        if isinstance(exception, HttpError):
+            return []
diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py
index 3cbe9f3b4..fe69d4bde 100644
--- a/scrapy/tests/test_spidermiddleware_httperror.py
+++ b/scrapy/tests/test_spidermiddleware_httperror.py
@@ -2,7 +2,7 @@ from unittest import TestCase
 
 from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
-from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException
+from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError
 
 
 class TestHttpErrorMiddleware(TestCase):
@@ -20,9 +20,17 @@ class TestHttpErrorMiddleware(TestCase):
     def test_process_spider_input(self):
         self.assertEquals(None,
                 self.mw.process_spider_input(self.res200, self.spider))
-        self.assertRaises(HttpErrorException,
+        self.assertRaises(HttpError,
                 self.mw.process_spider_input, self.res404, self.spider)
 
+    def test_process_spider_exception(self):
+        self.assertEquals([],
+                self.mw.process_spider_exception(self.res404, \
+                        HttpError(self.res404), self.spider))
+        self.assertEquals(None,
+                self.mw.process_spider_exception(self.res404, \
+                        Exception(), self.spider))
+
     def test_handle_httpstatus_list(self):
         res = self.res404.copy()
         res.request = Request('http://scrapytest.org',