Merge pull request #466 from kalessin/httperror

allow to use settings for defining http error handling defaults

scrapy/contrib/spidermiddleware/httperror.py

@@ -16,6 +16,14 @@ class HttpError(IgnoreRequest):
 
 class HttpErrorMiddleware(object):
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.handle_httpstatus_all = settings.getbool('HANDLE_HTTPSTATUS_ALL')
+        self.handle_httpstatus_list = settings.getlist('HANDLE_HTTPSTATUS_LIST')
+
     def process_spider_input(self, response, spider):
         if 200 <= response.status < 300: # common case
             return
@@ -24,8 +32,10 @@ class HttpErrorMiddleware(object):
             return
         if 'handle_httpstatus_list' in meta:
             allowed_statuses = meta['handle_httpstatus_list']
+        elif self.handle_httpstatus_all:
+            return
         else:
-            allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
+            allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list)
         if response.status in allowed_statuses:
             return
         raise HttpError(response, 'Ignoring non-200 response')

scrapy/tests/test_spidermiddleware_httperror.py

@@ -3,13 +3,14 @@ from unittest import TestCase
 from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
 from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError
+from scrapy.settings import Settings
 
 
 class TestHttpErrorMiddleware(TestCase):
 
     def setUp(self):
         self.spider = BaseSpider('foo')
-        self.mw = HttpErrorMiddleware()
+        self.mw = HttpErrorMiddleware(Settings({}))
         self.req = Request('http://scrapytest.org')
 
         self.res200 = Response('http://scrapytest.org', status=200)
@@ -42,3 +43,79 @@ class TestHttpErrorMiddleware(TestCase):
         self.assertEquals(None,
             self.mw.process_spider_input(self.res404, self.spider))
 
+class TestHttpErrorMiddlewareSettings(TestCase):
+    """Similar test, but with settings"""
+
+    def setUp(self):
+        self.spider = BaseSpider('foo')
+        self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_LIST': (402,)}))
+        self.req = Request('http://scrapytest.org')
+
+        self.res200 = Response('http://scrapytest.org', status=200)
+        self.res200.request = self.req
+        self.res404 = Response('http://scrapytest.org', status=404)
+        self.res404.request = self.req
+        self.res402 = Response('http://scrapytest.org', status=402)
+        self.res402.request = self.req
+
+    def test_process_spider_input(self):
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, self.res404, self.spider)
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res402, self.spider))
+
+    def test_meta_overrides_settings(self):
+        request = Request('http://scrapytest.org',
+                              meta={'handle_httpstatus_list': [404]})
+        res404 = self.res404.copy()
+        res404.request = request
+        res402 = self.res402.copy()
+        res402.request = request
+
+        self.assertEquals(None,
+            self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, res402, self.spider)
+
+    def test_spider_override_settings(self):
+        self.spider.handle_httpstatus_list = [404]
+        self.assertEquals(None,
+            self.mw.process_spider_input(self.res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, self.res402, self.spider)
+
+class TestHttpErrorMiddlewareHandleAll(TestCase):
+
+    def setUp(self):
+        self.spider = BaseSpider('foo')
+        self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_ALL': True}))
+        self.req = Request('http://scrapytest.org')
+
+        self.res200 = Response('http://scrapytest.org', status=200)
+        self.res200.request = self.req
+        self.res404 = Response('http://scrapytest.org', status=404)
+        self.res404.request = self.req
+        self.res402 = Response('http://scrapytest.org', status=402)
+        self.res402.request = self.req
+
+    def test_process_spider_input(self):
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res200, self.spider))
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res404, self.spider))
+
+    def test_meta_overrides_settings(self):
+        request = Request('http://scrapytest.org',
+                              meta={'handle_httpstatus_list': [404]})
+        res404 = self.res404.copy()
+        res404.request = request
+        res402 = self.res402.copy()
+        res402.request = request
+
+        self.assertEquals(None,
+            self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, res402, self.spider)
+