diff --git a/scrapy/trunk/scrapy/conf/core_settings.py b/scrapy/trunk/scrapy/conf/core_settings.py index 70da5c802..ad140af03 100644 --- a/scrapy/trunk/scrapy/conf/core_settings.py +++ b/scrapy/trunk/scrapy/conf/core_settings.py @@ -11,7 +11,7 @@ USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION) DOWNLOAD_TIMEOUT = 180 # 3mins CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain -CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid +CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires") LOG_ENABLED = True # LOGLEVEL = 'DEBUG' # default loglevel diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py index a97f7dae8..356c51127 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -1,6 +1,7 @@ from __future__ import with_statement import os +import sys import hashlib import datetime import urlparse @@ -33,7 +34,11 @@ class CacheMiddleware(object): key = request.fingerprint() domain = spider.domain_name - response = self.cache.retrieve_response(domain, key) + try: + response = self.cache.retrieve_response(domain, key) + except: + log.msg("Corrupt cache for %s" % request.url, log.WARNING) + response = False if response: response.cached = True if not 200 <= int(response.status) < 300: @@ -115,11 +120,16 @@ class Cache(object): if os.path.exists(requestpath): with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: metadata = pickle.load(f) - if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=settings.getint('CACHE2_EXPIRATION_SECS')): - return True + expiration_secs = settings.getint('CACHE2_EXPIRATION_SECS') + if expiration_secs >= 0: + if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs): + return True + else: + log.msg('dropping old cached response from %s' % metadata['timestamp']) + return False else: - log.msg('dropping old cached response from %s' % metadata['timestamp']) - return False + # disabled cache expiration + return True else: return False