From 341c5962320c03ed373c8eb58a5814954593ec95 Mon Sep 17 00:00:00 2001 From: deed02392 Date: Mon, 24 Feb 2014 15:55:04 +0000 Subject: [PATCH] Update httperror.py Log ignored HTTP pages when the response code is not allowed or not configured to be handled. --- scrapy/contrib/spidermiddleware/httperror.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py index 573770119..d5a26da63 100644 --- a/scrapy/contrib/spidermiddleware/httperror.py +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -4,7 +4,7 @@ HttpError Spider Middleware See documentation in docs/topics/spider-middleware.rst """ from scrapy.exceptions import IgnoreRequest - +from scrapy import log class HttpError(IgnoreRequest): """A non-200 response was filtered""" @@ -38,6 +38,9 @@ class HttpErrorMiddleware(object): allowed_statuses = getattr(spider, 'handle_httpstatus_list', self.handle_httpstatus_list) if response.status in allowed_statuses: return + log.msg(format="Ignoring HTTP response code: not handled or not allowed: %(status_code)d", + level=log.DEBUG, spider=spider, + status_code=response.status) raise HttpError(response, 'Ignoring non-200 response') def process_spider_exception(self, response, exception, spider):