From f87be371a224b163ddb119cff31e17f4dd628b36 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 21 Nov 2013 14:33:17 -0200 Subject: [PATCH] better names for HANDLE_* settings, and added doc --- docs/topics/spider-middleware.rst | 24 ++++++++++++++++++- scrapy/contrib/spidermiddleware/httperror.py | 4 ++-- .../tests/test_spidermiddleware_httperror.py | 4 ++-- 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 200e19d4a..332825337 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -214,7 +214,8 @@ status codes are in the 200-300 range. If you still want to process response codes outside that range, you can specify which response codes the spider is able to handle using the -``handle_httpstatus_list`` spider attribute. +``handle_httpstatus_list`` spider attribute or +:setting:`HTTPERROR_ALLOWED_CODES` setting. For example, if you want your spider to handle 404 responses you can do this:: @@ -235,6 +236,27 @@ For more information see: `HTTP Status Code Definitions`_. .. _HTTP Status Code Definitions: http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html +HttpErrorMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPERROR_ALLOWED_CODES + +HTTPERROR_ALLOWED_CODES +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``[]`` + +Pass all responses with non-200 status codes contained in this list. + +.. setting:: HTTPERROR_ALLOW_ALL + +HTTPERROR_ALLOW_ALL +^^^^^^^^^^^^^^^^^^^ + +Default: ``False`` + +Pass all responses, regardless of its status code. + OffsiteMiddleware ----------------- diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index fb63744c5..573770119 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -21,8 +21,8 @@ class HttpErrorMiddleware(object): return cls(crawler.settings) def __init__(self, settings): - self.handle_httpstatus_all = settings.getbool('HANDLE_HTTPSTATUS_ALL') - self.handle_httpstatus_list = settings.getlist('HANDLE_HTTPSTATUS_LIST') + self.handle_httpstatus_all = settings.getbool('HTTPERROR_ALLOW_ALL') + self.handle_httpstatus_list = settings.getlist('HTTPERROR_ALLOWED_CODES') def process_spider_input(self, response, spider): if 200 <= response.status < 300: # common case diff --git a/scrapy/tests/test_spidermiddleware_httperror.py b/scrapy/tests/test_spidermiddleware_httperror.py index d239ff895..0fb730c92 100644 --- a/scrapy/tests/test_spidermiddleware_httperror.py +++ b/scrapy/tests/test_spidermiddleware_httperror.py @@ -48,7 +48,7 @@ class TestHttpErrorMiddlewareSettings(TestCase): def setUp(self): self.spider = BaseSpider('foo') - self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_LIST': (402,)})) + self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOWED_CODES': (402,)})) self.req = Request('http://scrapytest.org') self.res200 = Response('http://scrapytest.org', status=200) @@ -90,7 +90,7 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): def setUp(self): self.spider = BaseSpider('foo') - self.mw = HttpErrorMiddleware(Settings({'HANDLE_HTTPSTATUS_ALL': True})) + self.mw = HttpErrorMiddleware(Settings({'HTTPERROR_ALLOW_ALL': True})) self.req = Request('http://scrapytest.org') self.res200 = Response('http://scrapytest.org', status=200)