From 55bee912a2adf6273a5aa4e9e158be452d9f49f9 Mon Sep 17 00:00:00 2001
From: Martin Olveyra <molveyra@gmail.com>
Date: Wed, 20 Nov 2013 20:12:49 -0200
Subject: [PATCH] allow to use settings for defining http error handling
 defaults

---
 scrapy/contrib/spidermiddleware/httperror.py  | 12 ++-
 .../tests/test_spidermiddleware_httperror.py  | 79 ++++++++++++++++++-
 2 files changed, 89 insertions(+), 2 deletions(-)

diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py
index 5ffb2dadb..fb63744c5 100644
--- a/scrapy/contrib/spidermiddleware/httperror.py
+++ b/scrapy/contrib/spidermiddleware/httperror.py
@@ -16,6 +16,14 @@ class HttpError(IgnoreRequest):
 
 class HttpErrorMiddleware(object):
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler.settings)
+
+    def __init__(self, settings):
+        self.handle_httpstatus_all = settings.getbool('HANDLE_HTTPSTATUS_ALL')
+        self.handle_httpstatus_list = settings.getlist('HANDLE_HTTPSTATUS_LIST')
+
     def process_spider_input(self, response, spider):
         if 200 <= response.status < 300: # common case
             return
@@ -24,8 +32,10 @@ class HttpErrorMiddleware(object):
             return
         if 'handle_httpstatus_list' in meta:
             allowed_statuses = meta['handle_httpstatus_list']
+        elif self.handle_httpstatus_all:
+            return
         else:
-            allowed_statuses = getattr(spider, 'handle_httpstatus_list', ())
+            allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list)
         if response.status in allowed_statuses:
             return
         raise HttpError(response, 'Ignoring non-200 response')
diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py
index fe69d4bde..d239ff895 100644
--- a/scrapy/tests/test_spidermiddleware_httperror.py
+++ b/scrapy/tests/test_spidermiddleware_httperror.py
@@ -3,13 +3,14 @@ from unittest import TestCase
 from scrapy.http import Response, Request
 from scrapy.spider import BaseSpider
 from scrapy.contrib.spidermiddleware.httperror import HttpErrorMiddleware, HttpError
+from scrapy.settings import Settings
 
 
 class TestHttpErrorMiddleware(TestCase):
 
     def setUp(self):
         self.spider = BaseSpider('foo')
-        self.mw = HttpErrorMiddleware()
+        self.mw = HttpErrorMiddleware(Settings({}))
         self.req = Request('http://scrapytest.org')
 
         self.res200 = Response('http://scrapytest.org', status=200)
@@ -42,3 +43,79 @@ class TestHttpErrorMiddleware(TestCase):
         self.assertEquals(None,
             self.mw.process_spider_input(self.res404, self.spider))
 
+class TestHttpErrorMiddlewareSettings(TestCase):
+    """Similar test, but with settings"""
+
+    def setUp(self):
+        self.spider = BaseSpider('foo')
+        self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_LIST': (402,)}))
+        self.req = Request('http://scrapytest.org')
+
+        self.res200 = Response('http://scrapytest.org', status=200)
+        self.res200.request = self.req
+        self.res404 = Response('http://scrapytest.org', status=404)
+        self.res404.request = self.req
+        self.res402 = Response('http://scrapytest.org', status=402)
+        self.res402.request = self.req
+
+    def test_process_spider_input(self):
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res200, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, self.res404, self.spider)
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res402, self.spider))
+
+    def test_meta_overrides_settings(self):
+        request = Request('http://scrapytest.org',
+                              meta={'handle_httpstatus_list': [404]})
+        res404 = self.res404.copy()
+        res404.request = request
+        res402 = self.res402.copy()
+        res402.request = request
+
+        self.assertEquals(None,
+            self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, res402, self.spider)
+
+    def test_spider_override_settings(self):
+        self.spider.handle_httpstatus_list = [404]
+        self.assertEquals(None,
+            self.mw.process_spider_input(self.res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, self.res402, self.spider)
+
+class TestHttpErrorMiddlewareHandleAll(TestCase):
+
+    def setUp(self):
+        self.spider = BaseSpider('foo')
+        self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_ALL': True}))
+        self.req = Request('http://scrapytest.org')
+
+        self.res200 = Response('http://scrapytest.org', status=200)
+        self.res200.request = self.req
+        self.res404 = Response('http://scrapytest.org', status=404)
+        self.res404.request = self.req
+        self.res402 = Response('http://scrapytest.org', status=402)
+        self.res402.request = self.req
+
+    def test_process_spider_input(self):
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res200, self.spider))
+        self.assertEquals(None,
+                self.mw.process_spider_input(self.res404, self.spider))
+
+    def test_meta_overrides_settings(self):
+        request = Request('http://scrapytest.org',
+                              meta={'handle_httpstatus_list': [404]})
+        res404 = self.res404.copy()
+        res404.request = request
+        res402 = self.res402.copy()
+        res402.request = request
+
+        self.assertEquals(None,
+            self.mw.process_spider_input(res404, self.spider))
+        self.assertRaises(HttpError,
+                self.mw.process_spider_input, res402, self.spider)
+