From c3b2cabf6c6600a5a2c6bbef2035ac7616ef6a06 Mon Sep 17 00:00:00 2001 From: Jamey Sharp Date: Sun, 28 Dec 2014 20:04:36 -0800 Subject: [PATCH] Allow setting RFC2616Policy to cache unconditionally. A spider may wish to have all responses available in the cache, for future use with "Cache-Control: max-stale", for instance. The DummyPolicy caches all responses but never revalidates them, and sometimes a more nuanced policy is desirable. This setting still respects "Cache-Control: no-store" directives in responses. If you don't want that, filter "no-store" out of the Cache-Control headers in responses you feed to the cache middleware. --- scrapy/extensions/httpcache.py | 4 ++++ scrapy/settings/default_settings.py | 1 + 2 files changed, 5 insertions(+) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index c0efb8996..4276ec928 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -38,6 +38,7 @@ class RFC2616Policy(object): MAXAGE = 3600 * 24 * 365 # one year def __init__(self, settings): + self.always_store = settings.getbool('HTTPCACHE_ALWAYS_STORE') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_response_cache_controls = settings.getlist('HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS') self._cc_parsed = WeakKeyDictionary() @@ -73,6 +74,9 @@ class RFC2616Policy(object): # Never cache 304 (Not Modified) responses elif response.status == 304: return False + # Cache unconditionally if configured to do so + elif self.always_store: + return True # Any hint on response expiration is good elif 'max-age' in cc or 'Expires' in response.headers: return True diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index bd1bb0936..5f9f4b98e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -153,6 +153,7 @@ HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 +HTTPCACHE_ALWAYS_STORE = False HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []