From 81a0e3cd93954ecd0a728ef5fcf5dc92a281fdd5 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 20 Sep 2016 13:42:28 +0200 Subject: [PATCH] Raise log level for HttpErrorMiddleware to INFO (from DEBUG) Fixes GH-910 --- scrapy/spidermiddlewares/httperror.py | 2 +- tests/test_spidermiddleware_httperror.py | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index 6b7c50fce..e34c265c1 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -46,7 +46,7 @@ class HttpErrorMiddleware(object): def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): - logger.debug( + logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {'response': response}, extra={'spider': spider}, ) diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index a64400482..319746350 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -1,3 +1,4 @@ +import logging from unittest import TestCase from testfixtures import LogCapture @@ -185,3 +186,29 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): self.assertIn('Ignoring response <500', str(log)) self.assertNotIn('Ignoring response <200', str(log)) self.assertNotIn('Ignoring response <402', str(log)) + + @defer.inlineCallbacks + def test_logging_level(self): + # HttpError logs ignored responses with level INFO + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.INFO) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertIn('Ignoring response <402', str(log)) + self.assertIn('Ignoring response <404', str(log)) + self.assertIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log)) + + # with level WARNING, we shouldn't capture anything from HttpError + crawler = get_crawler(_HttpErrorSpider) + with LogCapture(level=logging.WARNING) as log: + yield crawler.crawl() + self.assertEqual(crawler.spider.parsed, {'200'}) + self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + + self.assertNotIn('Ignoring response <402', str(log)) + self.assertNotIn('Ignoring response <404', str(log)) + self.assertNotIn('Ignoring response <500', str(log)) + self.assertNotIn('Ignoring response <200', str(log))