diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index bd112c48d..521327bfe 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,11 +1,21 @@ from email.utils import formatdate +from twisted.internet import defer +from twisted.internet.error import TimeoutError, DNSLookupError, \ + ConnectionRefusedError, ConnectionDone, ConnectError, \ + ConnectionLost, TCPTimedOutError from scrapy import signals from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object +from scrapy.xlib.tx import ResponseFailed class HttpCacheMiddleware(object): + DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, + ConnectionRefusedError, ConnectionDone, ConnectError, + ConnectionLost, TCPTimedOutError, ResponseFailed, + IOError) + def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured @@ -84,6 +94,12 @@ class HttpCacheMiddleware(object): self._cache_response(spider, response, request, cachedresponse) return response + def process_exception(self, request, exception, spider): + cachedresponse = request.meta.pop('cached_response', None) + if cachedresponse is not None and isinstance(exception, self.DOWNLOAD_EXCEPTIONS): + self.stats.inc_value('httpcache/errorrecovery', spider=spider) + return cachedresponse + def _cache_response(self, spider, response, request, cachedresponse): if self.policy.should_cache_response(response, request): self.stats.inc_value('httpcache/store', spider=spider) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 3173656fe..8011581ac 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -99,6 +99,14 @@ class RFC2616Policy(object): return False def is_cached_response_valid(self, cachedresponse, response, request): + # Use the cached response if the new response is a server error, + # as long as the old response didn't specify must-revalidate. + if response.status >= 500: + cc = self._parse_cachecontrol(cachedresponse) + if 'must-revalidate' not in cc: + return True + + # Use the cached response if the server says it hasn't changed. return response.status == 304 def _set_conditional_validators(self, request, cachedresponse):