diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index c0efb8996..4276ec928 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -38,6 +38,7 @@ class RFC2616Policy(object): MAXAGE = 3600 * 24 * 365 # one year def __init__(self, settings): + self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') self._cc_parsed = WeakKeyDictionary() @@ -73,6 +74,9 @@ class RFC2616Policy(object): # Never cache 304 (Not Modified) responses elif response.status == 304: return False + # Cache unconditionally if configured to do so + elif self.always_store: + return True # Any hint on response expiration is good elif 'max-age' in cc or 'Expires' in response.headers: return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index bd1bb0936..5f9f4b98e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -153,6 +153,7 @@ HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []