diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0c5dcc675..2889378cd 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -64,19 +64,18 @@ single Python class that defines one or more of the following methods: This method is called for each response that goes through the spider middleware and into the spider, for processing. - :meth:`process_spider_input` should return either ``None`` or an - iterable of :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` - objects. + :meth:`process_spider_input` should return ``None`` or raise and + exception. - If returns ``None``, Scrapy will continue processing this response, + If it returns ``None``, Scrapy will continue processing this response, executing all other middlewares until, finally, the response is handled to the spider for processing. - If returns an iterable, Scrapy won't bother calling any other spider - middleware :meth:`process_spider_input` and will return the iterable - back in the other direction for the :meth:`process_spider_output` to - process it, or the :meth:`process_spider_exception` if it raised an - exception. + If it raises an exception, Scrapy won't bother calling any other spider + middleware :meth:`process_spider_input` and will call the request + errback. The output of the errback is chained back in the other + direction for :meth:`process_spider_output` to process it, or + :meth:`process_spider_exception` if it raised an exception. :param reponse: the response being processed :type response: :class:`~scrapy.http.Response` object @@ -89,7 +88,7 @@ single Python class that defines one or more of the following methods: This method is called with the results returned from the Spider, after it has processed the response. - + :meth:`process_spider_output` must return an iterable of :class:`~scrapy.http.Request` or :class:`~scrapy.item.Item` objects. diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index 78cecfdf7..6da26aa2d 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -3,6 +3,16 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ +from scrapy.core.exceptions import IgnoreRequest + + +class HttpErrorException(IgnoreRequest): + """A non-200 response was filtered""" + + def __init__(self, response, *args, **kwargs): + self.response = response + super(HttpErrorException, self).__init__(*args, **kwargs) + class HttpErrorMiddleware(object): @@ -15,4 +25,5 @@ class HttpErrorMiddleware(object): allowed_statuses = getattr(spider, 'handle_httpstatus_list', ()) if response.status in allowed_statuses: return - return [] + raise HttpErrorException(response, 'Ignoring non-200 response') + diff --git a/scrapy/spider/middleware.py b/scrapy/spider/middleware.py index 416b3cc9b..4a8342c0a 100644 --- a/scrapy/spider/middleware.py +++ b/scrapy/spider/middleware.py @@ -7,11 +7,11 @@ docs/topics/spider-middleware.rst """ from scrapy import log +from twisted.python.failure import Failure from scrapy.core.exceptions import NotConfigured from scrapy.utils.misc import load_object from scrapy.utils.conf import build_component_list from scrapy.utils.defer import mustbe_deferred -from scrapy.http import Request from scrapy.conf import settings def _isiterable(possible_iterator): @@ -60,12 +60,14 @@ class SpiderMiddlewareManager(object): def process_spider_input(response): for method in self.spider_middleware: - result = method(response=response, spider=spider) - assert result is None or _isiterable(result), \ - 'Middleware %s must returns None or an iterable object, got %s ' % \ - (fname(method), type(result)) - if result is not None: - return result + try: + result = method(response=response, spider=spider) + assert result is None, \ + 'Middleware %s must returns None or ' \ + 'raise an exception, got %s ' \ + % (fname(method), type(result)) + except: + return scrape_func(Failure(), request, spider) return scrape_func(response, request, spider) def process_spider_exception(_failure): diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index 505b1a091..3cbe9f3b4 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -2,7 +2,7 @@ from unittest import TestCase from scrapy.http import Response, Request from scrapy.spider import BaseSpider -from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware +from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpErrorException class TestHttpErrorMiddleware(TestCase): @@ -18,21 +18,19 @@ class TestHttpErrorMiddleware(TestCase): self.res404.request = self.req def test_process_spider_input(self): - self.assertEquals(self.mw.process_spider_input(self.res200, self.spider), - None) - - self.assertEquals(self.mw.process_spider_input(self.res404, self.spider), - []) + self.assertEquals(None, + self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpErrorException, + self.mw.process_spider_input, self.res404, self.spider) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - - self.assertEquals(self.mw.process_spider_input(res, self.spider), - None) + self.assertEquals(None, + self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEquals(self.mw.process_spider_input(self.res404, self.spider), - None) + self.assertEquals(None, + self.mw.process_spider_input(self.res404, self.spider))