diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index 6da26aa2d..b76b7494b 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -6,12 +6,12 @@ See documentation in docs/topics/spider-middleware.rst from scrapy.core.exceptions import IgnoreRequest -class HttpErrorException(IgnoreRequest): +class HttpError(IgnoreRequest): """A non-200 response was filtered""" def __init__(self, response, *args, **kwargs): self.response = response - super(HttpErrorException, self).__init__(*args, **kwargs) + super(HttpError, self).__init__(*args, **kwargs) class HttpErrorMiddleware(object): @@ -25,5 +25,8 @@ class HttpErrorMiddleware(object): allowed_statuses = getattr(spider, 'handle_httpstatus_list', ()) if response.status in allowed_statuses: return - raise HttpErrorException(response, 'Ignoring non-200 response') + raise HttpError(response, 'Ignoring non-200 response') + def process_spider_exception(self, response, exception, spider): + if isinstance(exception, HttpError): + return [] diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index 3cbe9f3b4..fe69d4bde 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import BaseSpider -from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException +from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError class TestHttpErrorMiddleware(TestCase): @@ -20,9 +20,17 @@ class TestHttpErrorMiddleware(TestCase): def test_process_spider_input(self): self.assertEquals(None, self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpErrorException, + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + def test_process_spider_exception(self): + self.assertEquals([], + self.mw.process_spider_exception(self.res404, \ + HttpError(self.res404), self.spider)) + self.assertEquals(None, + self.mw.process_spider_exception(self.res404, \ + Exception(), self.spider)) + def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org',