RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

HttpErrorMiddleware stats

This commit is contained in:
Mikhail Korobov 2017-02-16 03:11:28 +05:00
parent cac51190df
commit 26c1256f95
2 changed files with 14 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/spidermiddlewares/httperror.py
View File

@ -46,6 +46,10 @@ class HttpErrorMiddleware(object):
def process_spider_exception(self, response, exception, spider):
if isinstance(exception, HttpError):
spider.crawler.stats.inc_value('httperror/response_ignored_count')
spider.crawler.stats.inc_value(
'httperror/response_ignored_status_count/%s' % response.status
)
logger.info(
"Ignoring response %(response)r: HTTP status code is not handled or not allowed",
{'response': response}, extra={'spider': spider},

13
tests/test_spidermiddleware_httperror.py
View File

@ -60,7 +60,8 @@ def _responses(request, status_codes):
class TestHttpErrorMiddleware(TestCase):
def setUp(self):
self.spider = Spider('foo')
crawler = get_crawler(Spider)
self.spider = Spider.from_crawler(crawler, name='foo')
self.mw = HttpErrorMiddleware(Settings({}))
self.req = Request('http://scrapytest.org')
self.res200, self.res404 = _responses(self.req, [200, 404])
@ -73,10 +74,10 @@ class TestHttpErrorMiddleware(TestCase):
def test_process_spider_exception(self):
self.assertEquals([],
self.mw.process_spider_exception(self.res404, \
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEquals(None,
self.mw.process_spider_exception(self.res404, \
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
def test_handle_httpstatus_list(self):
@ -173,6 +174,12 @@ class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
self.assertEqual(crawler.spider.parsed, {'200'})
self.assertEqual(crawler.spider.failed, {'404', '402', '500'})
get_value = crawler.stats.get_value
self.assertEqual(get_value('httperror/response_ignored_count'), 3)
self.assertEqual(get_value('httperror/response_ignored_status_count/404'), 1)
self.assertEqual(get_value('httperror/response_ignored_status_count/402'), 1)
self.assertEqual(get_value('httperror/response_ignored_status_count/500'), 1)
@defer.inlineCallbacks
def test_logging(self):
crawler = get_crawler(_HttpErrorSpider)