From 19915504422ccf6735b11f176ae8170c43562c29 Mon Sep 17 00:00:00 2001 From: Jamey Sharp Date: Mon, 29 Dec 2014 14:06:04 -0800 Subject: [PATCH] Allow client to bound max-age for revalidation. Unlike specifying "Cache-Control: no-cache", if the request specifies "max-age=0", then the cached validators will be used if possible to avoid re-fetching unchanged pages. That said, it's still useful to be able to specify "no-cache" on the request, in cases where the origin server may have changed page contents without changing validators. --- scrapy/extensions/httpcache.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 4276ec928..f33fcf819 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -101,6 +101,11 @@ class RFC2616Policy(object): now = time() freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) currentage = self._compute_current_age(cachedresponse, request, now) + + reqmaxage = self._get_max_age(ccreq) + if reqmaxage is not None: + freshnesslifetime = min(freshnesslifetime, reqmaxage) + if currentage < freshnesslifetime: return True @@ -144,15 +149,19 @@ class RFC2616Policy(object): if 'ETag' in cachedresponse.headers: request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + def _get_max_age(self, cc): + try: + return max(0, int(cc['max-age'])) + except (KeyError, ValueError): + return None + def _compute_freshness_lifetime(self, response, request, now): # Reference nsHttpResponseHead::ComputeFreshnessLifetime # http://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#410 cc = self._parse_cachecontrol(response) - if 'max-age' in cc: - try: - return max(0, int(cc['max-age'])) - except ValueError: - pass + maxage = self._get_max_age(cc) + if maxage is not None: + return maxage # Parse date header or synthesize it if none exists date = rfc1123_to_epoch(response.headers.get('Date')) or now