diff --git a/scrapy/conf/default_settings.py b/scrapy/conf/default_settings.py index f716ecaf4..866124345 100644 --- a/scrapy/conf/default_settings.py +++ b/scrapy/conf/default_settings.py @@ -59,7 +59,6 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.contrib.downloadermiddleware.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.contrib.downloadermiddleware.errorpages.ErrorPagesMiddleware': 200, 'scrapy.contrib.downloadermiddleware.httpauth.HttpAuthMiddleware': 300, 'scrapy.contrib.downloadermiddleware.useragent.UserAgentMiddleware': 400, 'scrapy.contrib.downloadermiddleware.retry.RetryMiddleware': 500, @@ -170,6 +169,7 @@ SPIDER_MIDDLEWARES = {} SPIDER_MIDDLEWARES_BASE = { # Engine side + 'scrapy.contrib.spidermiddleware.httperror.HttpErrorMiddleware': 50, 'scrapy.contrib.itemsampler.ItemSamplerMiddleware': 100, 'scrapy.contrib.spidermiddleware.limit.RequestLimitMiddleware': 200, 'scrapy.contrib.spidermiddleware.restrict.RestrictMiddleware': 300, diff --git a/scrapy/contrib/downloadermiddleware/errorpages.py b/scrapy/contrib/downloadermiddleware/errorpages.py index e16233fa2..a665cbf81 100644 --- a/scrapy/contrib/downloadermiddleware/errorpages.py +++ b/scrapy/contrib/downloadermiddleware/errorpages.py @@ -1,4 +1,4 @@ -from scrapy.core.exceptions import HttpException +from scrapy.core.exceptions import IgnoreRequest from scrapy.utils.response import response_status_message class ErrorPagesMiddleware(object): @@ -13,5 +13,5 @@ class ErrorPagesMiddleware(object): if 200 <= status < 300 or status in getattr(spider, 'handle_httpstatus_list', []): return response else: - raise HttpException(status, None, response) + raise IgnoreRequest(response_status_message(status)) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index 0b97f947d..b8d4202a2 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -11,7 +11,6 @@ from scrapy import log from scrapy.stats import stats from scrapy.utils.misc import md5sum from scrapy.core.exceptions import DropItem, NotConfigured -from scrapy.core.exceptions import HttpException, IgnoreRequest from scrapy.conf import settings from scrapy.contrib.pipeline.media import MediaPipeline @@ -105,6 +104,9 @@ class BaseImagesPipeline(MediaPipeline): def media_to_download(self, request, info): def _onsuccess(result): + if not result: + return # returning None force download + last_modified = result.get('last_modified', None) if not last_modified: return # returning None force download diff --git a/scrapy/contrib/spidermiddleware/httperror.py b/scrapy/contrib/spidermiddleware/httperror.py new file mode 100644 index 000000000..77c7b36f4 --- /dev/null +++ b/scrapy/contrib/spidermiddleware/httperror.py @@ -0,0 +1,13 @@ + + +class HttpErrorMiddleware(object): + """Filter out response outside of a range of valid status codes + + This middleware filters out every response with status outside of the range 200<=status<300 + Spiders can add more exceptions using `handle_httpstatus_list` spider attribute. + """ + + def process_spider_input(self, response, spider): + if not (200 <= response.status < 300 or \ + response.status in getattr(spider, 'handle_httpstatus_list', [])): + return [] # skip response diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 9b139943e..78fc246c6 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -16,7 +16,7 @@ from scrapy.conf import settings from scrapy.core import signals from scrapy.core.scheduler import Scheduler, SchedulerMiddlewareManager from scrapy.core.downloader import Downloader -from scrapy.core.exceptions import IgnoreRequest, HttpException, DontCloseDomain +from scrapy.core.exceptions import IgnoreRequest, DontCloseDomain from scrapy.http import Response, Request from scrapy.item import ScrapedItem from scrapy.item.pipeline import ItemPipelineManager @@ -330,6 +330,7 @@ class ExecutionEngine(object): if self.debug_mode: log.msg('Downloading %s' % request_info(request), log.DEBUG) domain = spider.domain_name + referer = request.headers.get('Referer', None) def _on_success(response): """handle the result of a page download""" @@ -338,7 +339,7 @@ class ExecutionEngine(object): log.msg("Requested %s" % request_info(request), level=log.DEBUG, domain=domain) if isinstance(response, Response): response.request = request # tie request to obtained response - log.msg("Crawled %s from <%s>" % (response, request.headers.get('referer')), level=log.DEBUG, domain=domain) + log.msg("Crawled %s from <%s>" % (response, referer), level=log.DEBUG, domain=domain) return response elif isinstance(response, Request): redirected = response # proper alias @@ -349,11 +350,7 @@ class ExecutionEngine(object): def _on_error(_failure): """handle an error processing a page""" ex = _failure.value - if isinstance(ex, IgnoreRequest): - log.msg(_failure.getErrorMessage(), level=log.DEBUG, domain=domain) - return _failure - referer = request.headers.get('Referer', None) - errmsg = str(ex) if isinstance(ex, HttpException) else str(_failure) + errmsg = str(_failure) if not isinstance(ex, IgnoreRequest) else _failure.getErrorMessage() log.msg("Downloading <%s> from <%s>: %s" % (request.url, referer, errmsg), log.ERROR, domain=domain) return Failure(IgnoreRequest(str(ex))) diff --git a/scrapy/core/exceptions.py b/scrapy/core/exceptions.py index f1079d3fa..6b172c57d 100644 --- a/scrapy/core/exceptions.py +++ b/scrapy/core/exceptions.py @@ -21,20 +21,6 @@ class DontCloseDomain(Exception): """Request the domain not to be closed yet""" pass -class HttpException(Exception): - def __init__(self, status, message, response): - if not message: - from twisted.web import http - message = http.responses.get(int(status)) - - self.status = int(status) - self.message = message - self.response = response - Exception.__init__(self, status, message, response) - - def __str__(self): - return '%s %s' % (self.status, self.message) - # Items class DropItem(Exception):