diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py index e34c265c1..def697c2b 100644 --- a/scrapy/spidermiddlewares/httperror.py +++ b/scrapy/spidermiddlewares/httperror.py @@ -46,6 +46,10 @@ class HttpErrorMiddleware(object): def process_spider_exception(self, response, exception, spider): if isinstance(exception, HttpError): + spider.crawler.stats.inc_value('httperror/response_ignored_count') + spider.crawler.stats.inc_value( + 'httperror/response_ignored_status_count/%s' % response.status + ) logger.info( "Ignoring response %(response)r: HTTP status code is not handled or not allowed", {'response': response}, extra={'spider': spider}, diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 319746350..e1407e6b3 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -60,7 +60,8 @@ def _responses(request, status_codes): class TestHttpErrorMiddleware(TestCase): def setUp(self): - self.spider = Spider('foo') + crawler = get_crawler(Spider) + self.spider = Spider.from_crawler(crawler, name='foo') self.mw = HttpErrorMiddleware(Settings({})) self.req = Request('http://scrapytest.org') self.res200, self.res404 = _responses(self.req, [200, 404]) @@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase): def test_process_spider_exception(self): self.assertEquals([], - self.mw.process_spider_exception(self.res404, \ + self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) self.assertEquals(None, - self.mw.process_spider_exception(self.res404, \ + self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): @@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase): self.assertEqual(crawler.spider.parsed, {'200'}) self.assertEqual(crawler.spider.failed, {'404', '402', '500'}) + get_value = crawler.stats.get_value + self.assertEqual(get_value('httperror/response_ignored_count'), 3) + self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1) + self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1) + self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1) + @defer.inlineCallbacks def test_logging(self): crawler = get_crawler(_HttpErrorSpider)