From b2d3f4dd1b8d1618a727c3670d1252b7b6290b47 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 16:15:04 +0100 Subject: [PATCH] Add cache storage support for real HTTP caching. Add real HTTP caching unit tests for both middleware and cache storage. --- scrapy/contrib/httpcache.py | 142 +++++++++++++++++ .../test_downloadermiddleware_httpcache.py | 148 +++++++++++++++++- 2 files changed, 289 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 623fc2bed..e815db3bc 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,8 +1,12 @@ import os +import calendar +import email.utils from time import time import cPickle as pickle +from scrapy import log from scrapy.http import Headers +from scrapy.exceptions import IgnoreRequest from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path @@ -59,3 +63,141 @@ class DbmCacheStorage(object): def _request_key(self, request): return request_fingerprint(request) + + +class BaseRealCacheStorage(object): + """ + Most of the code was taken from the httplib2 (MIT License) + https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py + """ + + def parse_cache_control(self, headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + + + def get(self, response_headers, request_headers): + """Determine freshness from the Date, Expires and Cache-Control headers. + + We don't handle the following: + + 1. Cache-Control: max-stale + 2. Age: headers are not used in the calculations. + + Not that this algorithm is simpler than you might think + because we are operating as a private (non-shared) cache. + This lets us ignore 's-maxage'. We can also ignore + 'proxy-invalidate' since we aren't a proxy. + We will never return a stale document as + fresh as a design decision, and thus the non-implementation + of 'max-stale'. This also lets us safely ignore 'must-revalidate' + since we operate as if every server has sent 'must-revalidate'. + Since we are private we get to ignore both 'public' and + 'private' parameters. We also ignore 'no-transform' since + we don't do any transformations. + The 'no-store' parameter is handled at a higher level. + So the only Cache-Control parameters we look at are: + + no-cache + only-if-cached + max-age + min-fresh + """ + + retval = "STALE" + cc = self.parse_cache_control(request_headers) + cc_response = self.parse_cache_control(response_headers) + + if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: + retval = "TRANSPARENT" + if 'cache-control' not in request_headers: + request_headers['cache-control'] = 'no-cache' + elif cc.has_key('no-cache'): + retval = "TRANSPARENT" + elif cc_response.has_key('no-cache'): + retval = "STALE" + elif cc.has_key('only-if-cached'): + retval = "FRESH" + elif response_headers.has_key('date'): + date = calendar.timegm(email.utils.parsedate_tz(response_headers['date'])) + now = time() + current_age = max(0, now - date) + if cc_response.has_key('max-age'): + try: + freshness_lifetime = int(cc_response['max-age']) + except ValueError: + freshness_lifetime = 0 + elif response_headers.has_key('expires'): + expires = email.utils.parsedate_tz(response_headers['expires']) + if None == expires: + freshness_lifetime = 0 + else: + freshness_lifetime = max(0, calendar.timegm(expires) - date) + else: + freshness_lifetime = 0 + if cc.has_key('max-age'): + try: + freshness_lifetime = int(cc['max-age']) + except ValueError: + freshness_lifetime = 0 + if cc.has_key('min-fresh'): + try: + min_fresh = int(cc['min-fresh']) + except ValueError: + min_fresh = 0 + current_age += min_fresh + if freshness_lifetime > current_age: + retval = "FRESH" + return retval + + def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''): + # Determine our course of action: + # Is the cached entry fresh or stale? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the Response object + # 2. [STALE] Update the Request object with cache validators if available + # 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache) + entry_disposition = self.get(Headers(response_headers), Headers(request.headers)) + + # Per the RFC, requests should not be repeated in these situations + if response_status in [400, 401, 403, 410]: + raise IgnoreRequest("Ignored request because cached response status is %d." % response_status) + + if entry_disposition == "FRESH": + log.msg("Cache is FRESH", level=log.DEBUG, spider=spider) + + headers = Headers(response_headers) + respcls = responsetypes.from_args(headers=headers, url=response_url) + response = respcls(url=response_url, headers=headers, status=response_status, body=response_body) + + return response + + new_request = request.copy() + if entry_disposition == "STALE": + log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider) + if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers: + new_request.headers['If-None-Match'] = response_headers['ETag'] + if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers: + new_request.headers['If-Modified-Since'] = response_headers['Last-Modified'] + elif entry_disposition == "TRANSPARENT": + log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider) + + return new_request + + +class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage): + def __init__(self, settings): + super(DbmRealCacheStorage, self).__init__(settings) + + def retrieve_response(self, spider, request): + data = self._read_data(spider, request) + if data is None: + return # not cached + else: + return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body']) diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index fdabaa4ab..d4c2b3dd7 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -2,6 +2,7 @@ import time import tempfile import shutil import unittest +import email.utils from contextlib import contextmanager from scrapy.http import Response, HtmlResponse, Request @@ -11,11 +12,17 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler from scrapy.contrib.downloadermiddleware.httpcache import \ FilesystemCacheStorage, HttpCacheMiddleware +from scrapy.contrib.httpcache import DbmRealCacheStorage class HttpCacheMiddlewareTest(unittest.TestCase): storage_class = FilesystemCacheStorage + realcache_storage_class = DbmRealCacheStorage + + yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60) + now = email.utils.formatdate() + tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60) def setUp(self): self.crawler = get_crawler() @@ -34,6 +41,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def _get_settings(self, **new_settings): settings = { 'HTTPCACHE_ENABLED': True, + 'HTTPCACHE_USE_DUMMY': True, 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], @@ -44,7 +52,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): @contextmanager def _storage(self, **new_settings): settings = self._get_settings(**new_settings) - storage = self.storage_class(settings) + if settings.getbool('HTTPCACHE_USE_DUMMY'): + storage = self.storage_class(settings) + else: + storage = self.realcache_storage_class(settings) storage.open_spider(self.spider) try: yield storage @@ -171,11 +182,146 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_real_http_cache_middleware_response304_not_cached(self): + # test response is not cached because the status is 304 Not Modified + # (so it should be cached already) + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + assert mw.process_request(self.request, self.spider) is None + response = Response('http://www.example.com', status=304) + mw.process_response(self.request, response, self.spider) + + assert 'cached' in response.flags + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + def test_real_http_cache_middleware_response_nostore_not_cached(self): + # test response is not cached because of the Cache-Control 'no-store' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + assert mw.process_request(self.request, self.spider) is None + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, + body='test body', status=200) + mw.process_response(self.request, response, self.spider) + + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + def test_real_http_cache_middleware_request_nostore_not_cached(self): + # test response is not cached because of the request's Cache-Control 'no-store' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + request = Request('http://www.example.com', + headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) + assert mw.process_request(request, self.spider) is None + mw.process_response(request, self.response, self.spider) + + assert mw.storage.retrieve_response(self.spider, request) is None + assert mw.process_request(request, self.spider) is None + + def test_real_http_cache_middleware_response_cached_and_fresh(self): + # test response cached and fresh + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + response = mw.process_response(self.request, self.response, self.spider) + self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) + assert 'cached' not in response.flags + + def test_real_http_cache_middleware_response_cached_and_stale(self): + # test response cached but stale + with self._middleware(HTTPCACHE_USE_DUMMY=False, + HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, + body='test body', status=200) + mw.process_response(self.request, response, self.spider) + assert mw.process_request(self.request, self.spider) is None + + response = mw.storage.retrieve_response(self.spider, self.request) + assert isinstance(response, Request) + + def test_real_http_cache_storage_response_cached_and_fresh(self): + # test response is cached and is fresh + # (response requested should be same as response received) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, + body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + self.assertEqualResponse(response, response2) + + def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self): + # test response is cached but further requests are ignored + # because response status is 403 (as per the RFC) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, + body='test body', status=403) + storage.store_response(self.spider, self.request, response) + self.assertRaises(IgnoreRequest, storage.retrieve_response, + self.spider, self.request) + + def test_real_http_cache_storage_response_cached_and_stale(self): + # test response is cached and is stale (no cache validators inserted) + # (request should be same as response received) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday}, + body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + assert isinstance(response2, Request) + self.assertEqualRequest(self.request, response2) + + def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self): + # test response is cached and is stale and cache validators are inserted + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday, + 'Last-Modified': self.yesterday}, body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + assert isinstance(response2, Request) + self.assertEqualRequestButWithCacheValidators(self.request, response2) + + def test_real_http_cache_storage_response_cached_and_transparent(self): + # test response is not cached because of the request's Cache-Control 'no-cache' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + request = Request('http://www.example.com', + headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) + assert storage.retrieve_response(self.spider, request) is None + storage.store_response(self.spider, request, self.response) + response = storage.retrieve_response(self.spider, request) + assert isinstance(response, Request) + self.assertEqualRequest(request, response) + def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) self.assertEqual(response1.status, response2.status) self.assertEqual(response1.headers, response2.headers) self.assertEqual(response1.body, response2.body) + def assertEqualRequest(self, request1, request2): + self.assertEqual(request1.url, request2.url) + self.assertEqual(request1.headers, request2.headers) + self.assertEqual(request1.body, request2.body) + + def assertEqualRequestButWithCacheValidators(self, request1, request2): + self.assertEqual(request1.url, request2.url) + assert not request1.headers.has_key('If-None-Match') + assert not request1.headers.has_key('If-Modified-Since') + assert (request2.headers.has_key('If-None-Match') or \ + request2.headers.has_key('If-Modified-Since')) + self.assertEqual(request1.body, request2.body) + if __name__ == '__main__': unittest.main()