From e396509f5b4a04f2979abe0fe4b6e7ab7cd15678 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 00:24:27 +0100 Subject: [PATCH 01/14] Removing plural from httpcache stats' value names. --- scrapy/contrib/downloadermiddleware/httpcache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index f0df8fbbd..cb12ac031 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -45,10 +45,10 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') - self.stats.inc_value('httpcache/hits', spider=spider) + self.stats.inc_value('httpcache/hit', spider=spider) return response - self.stats.inc_value('httpcache/misses', spider=spider) + self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) From bb55f39aedf682b5140c8b208468ddb6b02d3f85 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 01:52:45 +0100 Subject: [PATCH 02/14] Add middleware support for real HTTP caching. Add httpcache setting to allow either real or dummy HTTP caching (for backwards compatibility it's set to use dummy cache by default). --- .../contrib/downloadermiddleware/httpcache.py | 38 +++++++++++++++---- scrapy/settings/default_settings.py | 1 + 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index cb12ac031..e88371747 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -7,6 +7,7 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict from scrapy import signals from scrapy.http import Headers +from scrapy.http.request import Request from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint @@ -24,6 +25,7 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) + self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY') self.stats = stats @classmethod @@ -43,21 +45,41 @@ class HttpCacheMiddleware(object): if not self.is_cacheable(request): return response = self.storage.retrieve_response(spider, request) - if response and self.is_cacheable_response(response): - response.flags.append('cached') - self.stats.inc_value('httpcache/hit', spider=spider) - return response + # Response cached, but stale + if response and type(response) is Request: + # Return None so that Scrapy continues processing + self.stats.inc_value('httpcache/revalidation', spider=spider) + return + + if response and self.is_cacheable_response(response): + self.stats.inc_value('httpcache/hit', spider=spider) + if self.use_dummy_cache: + response.flags.append('cached') + return response + else: + # Response cached and fresh + raise IgnoreRequest("Ignored request already in cache: %s" % request) + + # Response not cached self.stats.inc_value('httpcache/miss', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): if (self.is_cacheable(request) - and self.is_cacheable_response(response) - and 'cached' not in response.flags): - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) + and self.is_cacheable_response(response)): + if self.use_dummy_cache: + if 'cached' not in response.flags: + self.storage.store_response(spider, request, response) + self.stats.inc_value('httpcache/store', spider=spider) + else: + if response.status != 304: + self.storage.store_response(spider, request, response) + self.stats.inc_value('httpcache/store', spider=spider) + else: + response.flags.append('cached') + self.stats.inc_value('httpcache/hits', spider=spider) return response def is_cacheable_response(self, response): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b70b2cadc..44b285f72 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -134,6 +134,7 @@ FEED_EXPORTERS_BASE = { } HTTPCACHE_ENABLED = False +HTTPCACHE_USE_DUMMY = True HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage' From 0e435fb5f9a5542abe3929f13a9b91059f2d6976 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 13:27:13 +0100 Subject: [PATCH 03/14] Add middleware support for the 'no-store' Cache-Control directive. --- scrapy/contrib/downloadermiddleware/httpcache.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index e88371747..b1da47586 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -83,11 +83,16 @@ class HttpCacheMiddleware(object): return response def is_cacheable_response(self, response): - return response.status not in self.ignore_http_codes + retval = response.status not in self.ignore_http_codes + if not self.use_dummy_cache and response.headers.has_key('cache-control'): + retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) + return retval def is_cacheable(self, request): - return urlparse_cached(request).scheme not in self.ignore_schemes - + retval = urlparse_cached(request).scheme not in self.ignore_schemes + if not self.use_dummy_cache and request.headers.has_key('cache-control'): + retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) + return retval class FilesystemCacheStorage(object): From b2d3f4dd1b8d1618a727c3670d1252b7b6290b47 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 16:15:04 +0100 Subject: [PATCH 04/14] Add cache storage support for real HTTP caching. Add real HTTP caching unit tests for both middleware and cache storage. --- scrapy/contrib/httpcache.py | 142 +++++++++++++++++ .../test_downloadermiddleware_httpcache.py | 148 +++++++++++++++++- 2 files changed, 289 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 623fc2bed..e815db3bc 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,8 +1,12 @@ import os +import calendar +import email.utils from time import time import cPickle as pickle +from scrapy import log from scrapy.http import Headers +from scrapy.exceptions import IgnoreRequest from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path @@ -59,3 +63,141 @@ class DbmCacheStorage(object): def _request_key(self, request): return request_fingerprint(request) + + +class BaseRealCacheStorage(object): + """ + Most of the code was taken from the httplib2 (MIT License) + https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py + """ + + def parse_cache_control(self, headers): + retval = {} + if headers.has_key('cache-control'): + parts = headers['cache-control'].split(',') + parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] + parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] + retval = dict(parts_with_args + parts_wo_args) + return retval + + + def get(self, response_headers, request_headers): + """Determine freshness from the Date, Expires and Cache-Control headers. + + We don't handle the following: + + 1. Cache-Control: max-stale + 2. Age: headers are not used in the calculations. + + Not that this algorithm is simpler than you might think + because we are operating as a private (non-shared) cache. + This lets us ignore 's-maxage'. We can also ignore + 'proxy-invalidate' since we aren't a proxy. + We will never return a stale document as + fresh as a design decision, and thus the non-implementation + of 'max-stale'. This also lets us safely ignore 'must-revalidate' + since we operate as if every server has sent 'must-revalidate'. + Since we are private we get to ignore both 'public' and + 'private' parameters. We also ignore 'no-transform' since + we don't do any transformations. + The 'no-store' parameter is handled at a higher level. + So the only Cache-Control parameters we look at are: + + no-cache + only-if-cached + max-age + min-fresh + """ + + retval = "STALE" + cc = self.parse_cache_control(request_headers) + cc_response = self.parse_cache_control(response_headers) + + if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: + retval = "TRANSPARENT" + if 'cache-control' not in request_headers: + request_headers['cache-control'] = 'no-cache' + elif cc.has_key('no-cache'): + retval = "TRANSPARENT" + elif cc_response.has_key('no-cache'): + retval = "STALE" + elif cc.has_key('only-if-cached'): + retval = "FRESH" + elif response_headers.has_key('date'): + date = calendar.timegm(email.utils.parsedate_tz(response_headers['date'])) + now = time() + current_age = max(0, now - date) + if cc_response.has_key('max-age'): + try: + freshness_lifetime = int(cc_response['max-age']) + except ValueError: + freshness_lifetime = 0 + elif response_headers.has_key('expires'): + expires = email.utils.parsedate_tz(response_headers['expires']) + if None == expires: + freshness_lifetime = 0 + else: + freshness_lifetime = max(0, calendar.timegm(expires) - date) + else: + freshness_lifetime = 0 + if cc.has_key('max-age'): + try: + freshness_lifetime = int(cc['max-age']) + except ValueError: + freshness_lifetime = 0 + if cc.has_key('min-fresh'): + try: + min_fresh = int(cc['min-fresh']) + except ValueError: + min_fresh = 0 + current_age += min_fresh + if freshness_lifetime > current_age: + retval = "FRESH" + return retval + + def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''): + # Determine our course of action: + # Is the cached entry fresh or stale? + # + # There seems to be three possible answers: + # 1. [FRESH] Return the Response object + # 2. [STALE] Update the Request object with cache validators if available + # 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache) + entry_disposition = self.get(Headers(response_headers), Headers(request.headers)) + + # Per the RFC, requests should not be repeated in these situations + if response_status in [400, 401, 403, 410]: + raise IgnoreRequest("Ignored request because cached response status is %d." % response_status) + + if entry_disposition == "FRESH": + log.msg("Cache is FRESH", level=log.DEBUG, spider=spider) + + headers = Headers(response_headers) + respcls = responsetypes.from_args(headers=headers, url=response_url) + response = respcls(url=response_url, headers=headers, status=response_status, body=response_body) + + return response + + new_request = request.copy() + if entry_disposition == "STALE": + log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider) + if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers: + new_request.headers['If-None-Match'] = response_headers['ETag'] + if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers: + new_request.headers['If-Modified-Since'] = response_headers['Last-Modified'] + elif entry_disposition == "TRANSPARENT": + log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider) + + return new_request + + +class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage): + def __init__(self, settings): + super(DbmRealCacheStorage, self).__init__(settings) + + def retrieve_response(self, spider, request): + data = self._read_data(spider, request) + if data is None: + return # not cached + else: + return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body']) diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index fdabaa4ab..d4c2b3dd7 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -2,6 +2,7 @@ import time import tempfile import shutil import unittest +import email.utils from contextlib import contextmanager from scrapy.http import Response, HtmlResponse, Request @@ -11,11 +12,17 @@ from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler from scrapy.contrib.downloadermiddleware.httpcache import \ FilesystemCacheStorage, HttpCacheMiddleware +from scrapy.contrib.httpcache import DbmRealCacheStorage class HttpCacheMiddlewareTest(unittest.TestCase): storage_class = FilesystemCacheStorage + realcache_storage_class = DbmRealCacheStorage + + yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60) + now = email.utils.formatdate() + tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60) def setUp(self): self.crawler = get_crawler() @@ -34,6 +41,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def _get_settings(self, **new_settings): settings = { 'HTTPCACHE_ENABLED': True, + 'HTTPCACHE_USE_DUMMY': True, 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], @@ -44,7 +52,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): @contextmanager def _storage(self, **new_settings): settings = self._get_settings(**new_settings) - storage = self.storage_class(settings) + if settings.getbool('HTTPCACHE_USE_DUMMY'): + storage = self.storage_class(settings) + else: + storage = self.realcache_storage_class(settings) storage.open_spider(self.spider) try: yield storage @@ -171,11 +182,146 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags + def test_real_http_cache_middleware_response304_not_cached(self): + # test response is not cached because the status is 304 Not Modified + # (so it should be cached already) + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + assert mw.process_request(self.request, self.spider) is None + response = Response('http://www.example.com', status=304) + mw.process_response(self.request, response, self.spider) + + assert 'cached' in response.flags + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + def test_real_http_cache_middleware_response_nostore_not_cached(self): + # test response is not cached because of the Cache-Control 'no-store' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + assert mw.process_request(self.request, self.spider) is None + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, + body='test body', status=200) + mw.process_response(self.request, response, self.spider) + + assert mw.storage.retrieve_response(self.spider, self.request) is None + assert mw.process_request(self.request, self.spider) is None + + def test_real_http_cache_middleware_request_nostore_not_cached(self): + # test response is not cached because of the request's Cache-Control 'no-store' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + request = Request('http://www.example.com', + headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) + assert mw.process_request(request, self.spider) is None + mw.process_response(request, self.response, self.spider) + + assert mw.storage.retrieve_response(self.spider, request) is None + assert mw.process_request(request, self.spider) is None + + def test_real_http_cache_middleware_response_cached_and_fresh(self): + # test response cached and fresh + with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + response = mw.process_response(self.request, self.response, self.spider) + self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) + assert 'cached' not in response.flags + + def test_real_http_cache_middleware_response_cached_and_stale(self): + # test response cached but stale + with self._middleware(HTTPCACHE_USE_DUMMY=False, + HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, + body='test body', status=200) + mw.process_response(self.request, response, self.spider) + assert mw.process_request(self.request, self.spider) is None + + response = mw.storage.retrieve_response(self.spider, self.request) + assert isinstance(response, Request) + + def test_real_http_cache_storage_response_cached_and_fresh(self): + # test response is cached and is fresh + # (response requested should be same as response received) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, + body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + self.assertEqualResponse(response, response2) + + def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self): + # test response is cached but further requests are ignored + # because response status is 403 (as per the RFC) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, + body='test body', status=403) + storage.store_response(self.spider, self.request, response) + self.assertRaises(IgnoreRequest, storage.retrieve_response, + self.spider, self.request) + + def test_real_http_cache_storage_response_cached_and_stale(self): + # test response is cached and is stale (no cache validators inserted) + # (request should be same as response received) + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday}, + body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + assert isinstance(response2, Request) + self.assertEqualRequest(self.request, response2) + + def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self): + # test response is cached and is stale and cache validators are inserted + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + assert storage.retrieve_response(self.spider, self.request) is None + + response = Response('http://www.example.com', headers= + {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday, + 'Last-Modified': self.yesterday}, body='test body', status=200) + storage.store_response(self.spider, self.request, response) + response2 = storage.retrieve_response(self.spider, self.request) + assert isinstance(response2, Request) + self.assertEqualRequestButWithCacheValidators(self.request, response2) + + def test_real_http_cache_storage_response_cached_and_transparent(self): + # test response is not cached because of the request's Cache-Control 'no-cache' directive + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 + with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + request = Request('http://www.example.com', + headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) + assert storage.retrieve_response(self.spider, request) is None + storage.store_response(self.spider, request, self.response) + response = storage.retrieve_response(self.spider, request) + assert isinstance(response, Request) + self.assertEqualRequest(request, response) + def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) self.assertEqual(response1.status, response2.status) self.assertEqual(response1.headers, response2.headers) self.assertEqual(response1.body, response2.body) + def assertEqualRequest(self, request1, request2): + self.assertEqual(request1.url, request2.url) + self.assertEqual(request1.headers, request2.headers) + self.assertEqual(request1.body, request2.body) + + def assertEqualRequestButWithCacheValidators(self, request1, request2): + self.assertEqual(request1.url, request2.url) + assert not request1.headers.has_key('If-None-Match') + assert not request1.headers.has_key('If-Modified-Since') + assert (request2.headers.has_key('If-None-Match') or \ + request2.headers.has_key('If-Modified-Since')) + self.assertEqual(request1.body, request2.body) + if __name__ == '__main__': unittest.main() From fdaa35f6e8dedf92fb43ade412dcfc03c221b759 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Mon, 24 Dec 2012 19:37:53 +0100 Subject: [PATCH 05/14] Updated the downloader middleware documentation to reflect changes introduced by the support for real HTTP caching. --- docs/topics/downloader-middleware.rst | 65 ++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 10 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 57b1fdc57..e42f85b64 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -293,19 +293,41 @@ HttpCacheMiddleware .. class:: HttpCacheMiddleware - This middleware provides low-level cache to all HTTP requests and responses. - Every request and its corresponding response are cached. When the same - request is seen again, the response is returned without transferring - anything from the Internet. + There are two types of caches: - The HTTP cache is useful for testing spiders faster (without having to wait for - downloads every time) and for trying your spider offline, when an Internet - connection is not available. + * Dummy cache - Scrapy ships with two storage backends for the HTTP cache middleware: + This middleware was designed as a dummy low-level cache to all HTTP + requests and responses, with no awareness of any HTTP Cache-Control + directives. Every request and its corresponding response are cached. + When the same request is seen again, the response is returned + without transferring anything from the Internet. - * :ref:`httpcache-dbm-backend` - * :ref:`httpcache-fs-backend` + The HTTP cache is useful for testing spiders faster (without having to + wait for downloads every time) and for trying your spider offline, when + an Internet connection is not available. The goal is to be able to + "replay" a spider run *exactly as it run before* and not to use HTTP + caching (to save bandwidth and speed up the crawl). + + Scrapy ships with two storage backends for the dummy HTTP cache middleware: + + * :ref:`httpcache-dbm-backend` + * :ref:`httpcache-fs-backend` + + * Real HTTP cache + + This middleware was designed as a real HTTP cache with HTTP Cache-Control + awareness, aimed at production and used in continuous runs to avoid + downloading unmodified data (to save bandwidth and speed up crawls). + + In order to use the real HTTP cache, set: + + * :setting:`HTTPCACHE_USE_DUMMY` to ``False`` + * :setting:`HTTPCACHE_STORAGE` to ``'scrapy.contrib.httpcache.DbmRealCacheStorage'`` + + Scrapy ships with one storage backend for the real HTTP cache middleware: + + * :ref:`httprealcache-dbm-backend` You can change the storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own backend. @@ -324,6 +346,20 @@ to ``scrapy.contrib.httpcache.DbmCacheStorage``. By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. +.. _httprealcache-dbm-backend: + +DBM storage backend (real HTTP cache) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +It inherits from both ``scrapy.contrib.httpcache.DbmCacheStorage`` and +``scrapy.contrib.httpcache.BaseRealCacheStorage``, the latter providing +HTTP Cache-Control awareness. To use it set :setting:`HTTPCACHE_STORAGE` +to ``scrapy.contrib.httpcache.DbmRealCacheStorage`` and +:setting:`HTTPCACHE_USE_DUMMY` to ``False``. + +If you need to create your own storage backend with HTTP Cache-Control awareness, +you can inherit from ``scrapy.contrib.httpcache.BaseRealCacheStorage``. + .. _httpcache-fs-backend: File system backend @@ -372,6 +408,15 @@ Whether the HTTP cache will be enabled. .. versionchanged:: 0.11 Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. +.. setting:: HTTPCACHE_USE_DUMMY + +HTTPCACHE_USE_DUMMY +^^^^^^^^^^^^^^^^^^^ + +Default: ``True`` + +Whether to use the dummy or the real HTTP cache. The default is set to ``True`` for backwards compatibility. + .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS From 93a1102189b5cc53b5816ba74568a0ab7e6f5e6f Mon Sep 17 00:00:00 2001 From: Hasnain Lakhani Date: Wed, 26 Dec 2012 16:29:48 -0800 Subject: [PATCH 06/14] Implemented policies for HTTP Cache --- docs/topics/downloader-middleware.rst | 23 +++++++++ .../contrib/downloadermiddleware/httpcache.py | 4 ++ scrapy/settings/default_settings.py | 2 + .../test_downloadermiddleware_httpcache.py | 48 +++++++++++++++++++ 4 files changed, 77 insertions(+) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index e42f85b64..48292e10c 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -495,6 +495,29 @@ Default: ``'anydbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. +HTTPCACHE_POLICY_REQUEST +^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.18 + +Default: ```lambda request: True``` + +A callback function used by the HTTP cache to decide whether a request +is cacheable. The function should take a :class:`~scrapy.http.Request` +object as a parameter and return ``True`` if a cached response can be returned; +or ``False`` if it should be fetched again. + +HTTPCACHE_POLICY_RESPONSE +^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. versionadded:: 0.18 + +Default: ```lambda response: True``` + +A callback function used by the HTTP cache to decide whether a response +is cacheable. The function should take a :class:`~scrapy.http.Response` +object as a parameter and return ``True`` if a response can be cached; +or ``False`` if it should not be stored in the cache. HttpCompressionMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b1da47586..a5b7e5f05 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -27,6 +27,8 @@ class HttpCacheMiddleware(object): self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY') self.stats = stats + self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST') + self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE') @classmethod def from_crawler(cls, crawler): @@ -86,12 +88,14 @@ class HttpCacheMiddleware(object): retval = response.status not in self.ignore_http_codes if not self.use_dummy_cache and response.headers.has_key('cache-control'): retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) + retval = retval and self.policy_response(response) return retval def is_cacheable(self, request): retval = urlparse_cached(request).scheme not in self.ignore_schemes if not self.use_dummy_cache and request.headers.has_key('cache-control'): retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) + retval = retval and self.policy_request(request) return retval class FilesystemCacheStorage(object): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 44b285f72..1dd88219c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -142,6 +142,8 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' +HTTPCACHE_POLICY_REQUEST = lambda request : True +HTTPCACHE_POLICY_RESPONSE = lambda response : True ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index d4c2b3dd7..176a9d86a 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -45,6 +45,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase): 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], + 'HTTPCACHE_POLICY_REQUEST': lambda request : True, + 'HTTPCACHE_POLICY_RESPONSE': lambda response : True, } settings.update(new_settings) return Settings(settings) @@ -303,6 +305,52 @@ class HttpCacheMiddlewareTest(unittest.TestCase): response = storage.retrieve_response(self.spider, request) assert isinstance(response, Request) self.assertEqualRequest(request, response) + + def test_http_cache_request_policy(self): + callback = lambda r: (r.url == 'http://www.example.com/') + + # Should be cached + req, res = Request('http://www.example.com/'), Response('http://www.example.com/') + with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # Should not be cached + req, res = Request('http://www.test.com/'), Response('http://www.test.com/') + with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None + + def test_http_cache_response_policy(self): + callback = lambda r: (r.url == 'http://www.example.com/') + + # Should be cached + req, res = Request('http://www.example.com/'), Response('http://www.example.com/') + with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + cached = mw.process_request(req, self.spider) + assert isinstance(cached, Response), type(cached) + self.assertEqualResponse(res, cached) + assert 'cached' in cached.flags + + # Should not be cached + req, res = Request('http://www.test.com/'), Response('http://www.test.com/') + with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: + assert mw.process_request(req, self.spider) is None + mw.process_response(req, res, self.spider) + + assert mw.storage.retrieve_response(self.spider, req) is None + assert mw.process_request(req, self.spider) is None def assertEqualResponse(self, response1, response2): self.assertEqual(response1.url, response2.url) From 63d0b9f8c8c09b3e9825df02d6333b8238871b78 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Fri, 28 Dec 2012 11:06:04 +0100 Subject: [PATCH 07/14] Remove plural from stat key. --- scrapy/contrib/downloadermiddleware/httpcache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index a5b7e5f05..5608ec2b9 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -81,7 +81,7 @@ class HttpCacheMiddleware(object): self.stats.inc_value('httpcache/store', spider=spider) else: response.flags.append('cached') - self.stats.inc_value('httpcache/hits', spider=spider) + self.stats.inc_value('httpcache/hit', spider=spider) return response def is_cacheable_response(self, response): From 3e31d068725ad7b85f2ccdfa63c96f0b248fe5a9 Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Fri, 28 Dec 2012 13:28:35 +0100 Subject: [PATCH 08/14] Implement single HTTP cache policy --- .../contrib/downloadermiddleware/httpcache.py | 56 +++++++++++-------- scrapy/settings/default_settings.py | 4 +- .../test_downloadermiddleware_httpcache.py | 26 ++++----- 3 files changed, 45 insertions(+), 41 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 5608ec2b9..b891c018c 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -16,23 +16,44 @@ from scrapy.utils.misc import load_object from scrapy.utils.project import data_path -class HttpCacheMiddleware(object): +class HttpCachePolicy(object): + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) + self.policy = settings.get('HTTPCACHE_POLICY') + + if self.policy == 'dummy': + self.use_dummy_cache = True + else: + self.use_dummy_cache = False + + def should_cache_response(self, response): + retval = response.status not in self.ignore_http_codes + if not self.use_dummy_cache and response.headers.has_key('cache-control'): + retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) + #retval = retval and self.policy_response(response) + return retval + + def should_cache_request(self, request): + retval = urlparse_cached(request).scheme not in self.ignore_schemes + if not self.use_dummy_cache and request.headers.has_key('cache-control'): + retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) + #retval = retval and self.policy_request(request) + return retval + +class HttpCacheMiddleware(HttpCachePolicy): def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY') self.stats = stats - self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST') - self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE') + super(HttpCacheMiddleware, self).__init__(settings) @classmethod def from_crawler(cls, crawler): - o = cls(crawler.settings, crawler.stats) + o = cls.from_settings(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o @@ -44,7 +65,7 @@ class HttpCacheMiddleware(object): self.storage.close_spider(spider) def process_request(self, request, spider): - if not self.is_cacheable(request): + if not self.should_cache_request(request): return response = self.storage.retrieve_response(spider, request) @@ -54,7 +75,7 @@ class HttpCacheMiddleware(object): self.stats.inc_value('httpcache/revalidation', spider=spider) return - if response and self.is_cacheable_response(response): + if response and self.should_cache_response(response): self.stats.inc_value('httpcache/hit', spider=spider) if self.use_dummy_cache: response.flags.append('cached') @@ -69,8 +90,8 @@ class HttpCacheMiddleware(object): raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if (self.is_cacheable(request) - and self.is_cacheable_response(response)): + if (self.should_cache_request(request) + and self.should_cache_response(response)): if self.use_dummy_cache: if 'cached' not in response.flags: self.storage.store_response(spider, request, response) @@ -84,19 +105,6 @@ class HttpCacheMiddleware(object): self.stats.inc_value('httpcache/hit', spider=spider) return response - def is_cacheable_response(self, response): - retval = response.status not in self.ignore_http_codes - if not self.use_dummy_cache and response.headers.has_key('cache-control'): - retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) - retval = retval and self.policy_response(response) - return retval - - def is_cacheable(self, request): - retval = urlparse_cached(request).scheme not in self.ignore_schemes - if not self.use_dummy_cache and request.headers.has_key('cache-control'): - retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) - retval = retval and self.policy_request(request) - return retval class FilesystemCacheStorage(object): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 1dd88219c..f494dc448 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -134,7 +134,6 @@ FEED_EXPORTERS_BASE = { } HTTPCACHE_ENABLED = False -HTTPCACHE_USE_DUMMY = True HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage' @@ -142,8 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY_REQUEST = lambda request : True -HTTPCACHE_POLICY_RESPONSE = lambda response : True +HTTPCACHE_POLICY = 'dummy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 176a9d86a..7d505e4f9 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -41,12 +41,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def _get_settings(self, **new_settings): settings = { 'HTTPCACHE_ENABLED': True, - 'HTTPCACHE_USE_DUMMY': True, 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY_REQUEST': lambda request : True, - 'HTTPCACHE_POLICY_RESPONSE': lambda response : True, + 'HTTPCACHE_POLICY': 'dummy' } settings.update(new_settings) return Settings(settings) @@ -54,7 +52,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): @contextmanager def _storage(self, **new_settings): settings = self._get_settings(**new_settings) - if settings.getbool('HTTPCACHE_USE_DUMMY'): + if settings.get('HTTPCACHE_POLICY') == 'dummy': storage = self.storage_class(settings) else: storage = self.realcache_storage_class(settings) @@ -187,7 +185,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response304_not_cached(self): # test response is not cached because the status is 304 Not Modified # (so it should be cached already) - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', status=304) mw.process_response(self.request, response, self.spider) @@ -199,7 +197,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response_nostore_not_cached(self): # test response is not cached because of the Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, @@ -212,7 +210,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_request_nostore_not_cached(self): # test response is not cached because of the request's Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) assert mw.process_request(request, self.spider) is None @@ -223,14 +221,14 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_middleware_response_cached_and_fresh(self): # test response cached and fresh - with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw: + with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: response = mw.process_response(self.request, self.response, self.spider) self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) assert 'cached' not in response.flags def test_real_http_cache_middleware_response_cached_and_stale(self): # test response cached but stale - with self._middleware(HTTPCACHE_USE_DUMMY=False, + with self._middleware(HTTPCACHE_POLICY='rfc2616', HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, @@ -244,7 +242,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_fresh(self): # test response is cached and is fresh # (response requested should be same as response received) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -257,7 +255,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self): # test response is cached but further requests are ignored # because response status is 403 (as per the RFC) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -270,7 +268,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_stale(self): # test response is cached and is stale (no cache validators inserted) # (request should be same as response received) - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -283,7 +281,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self): # test response is cached and is stale and cache validators are inserted - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -297,7 +295,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def test_real_http_cache_storage_response_cached_and_transparent(self): # test response is not cached because of the request's Cache-Control 'no-cache' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._storage(HTTPCACHE_USE_DUMMY=False) as storage: + with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) assert storage.retrieve_response(self.spider, request) is None From cf5f0203b782425aec470e603853885e2aec641c Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Fri, 28 Dec 2012 16:11:47 +0100 Subject: [PATCH 09/14] Instead of extending from HttpCachePolicy, following the same approach used for storage selection --- .gitignore | 1 + .../contrib/downloadermiddleware/httpcache.py | 59 +++++++++++-------- scrapy/settings/default_settings.py | 2 +- .../test_downloadermiddleware_httpcache.py | 47 ++++++++------- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/.gitignore b/.gitignore index f7f30b06f..2a329bc3a 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ dropin.cache docs/build *egg-info .tox +venv diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b891c018c..c267d1c8a 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -16,44 +16,51 @@ from scrapy.utils.misc import load_object from scrapy.utils.project import data_path -class HttpCachePolicy(object): +class DummyPolicy(object): def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - self.policy = settings.get('HTTPCACHE_POLICY') + + def should_cache_response(self, response): + return response.status not in self.ignore_http_codes - if self.policy == 'dummy': - self.use_dummy_cache = True - else: - self.use_dummy_cache = False + def should_cache_request(self, request): + return urlparse_cached(request).scheme not in self.ignore_schemes + + +class RFC2616Policy(DummyPolicy): + def __init__(self, settings): + super(RFC2616Policy, self).__init__(settings) def should_cache_response(self, response): - retval = response.status not in self.ignore_http_codes - if not self.use_dummy_cache and response.headers.has_key('cache-control'): + retval = super(RFC2616Policy, self).should_cache_response(response) + + if response.headers.has_key('cache-control'): retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) #retval = retval and self.policy_response(response) return retval def should_cache_request(self, request): - retval = urlparse_cached(request).scheme not in self.ignore_schemes - if not self.use_dummy_cache and request.headers.has_key('cache-control'): + retval = super(RFC2616Policy, self).should_cache_request(request) + + if request.headers.has_key('cache-control'): retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) #retval = retval and self.policy_request(request) return retval -class HttpCacheMiddleware(HttpCachePolicy): +class HttpCacheMiddleware(object): def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') + self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) self.stats = stats - super(HttpCacheMiddleware, self).__init__(settings) @classmethod def from_crawler(cls, crawler): - o = cls.from_settings(crawler.settings, crawler.stats) + o = cls(crawler.settings, crawler.stats) crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o @@ -65,7 +72,7 @@ class HttpCacheMiddleware(HttpCachePolicy): self.storage.close_spider(spider) def process_request(self, request, spider): - if not self.should_cache_request(request): + if not self.policy.should_cache_request(request): return response = self.storage.retrieve_response(spider, request) @@ -75,14 +82,14 @@ class HttpCacheMiddleware(HttpCachePolicy): self.stats.inc_value('httpcache/revalidation', spider=spider) return - if response and self.should_cache_response(response): + if response and self.policy.should_cache_response(response): self.stats.inc_value('httpcache/hit', spider=spider) - if self.use_dummy_cache: - response.flags.append('cached') - return response - else: + if isinstance(self.policy, RFC2616Policy): # Response cached and fresh raise IgnoreRequest("Ignored request already in cache: %s" % request) + else: + response.flags.append('cached') + return response # Response not cached self.stats.inc_value('httpcache/miss', spider=spider) @@ -90,19 +97,19 @@ class HttpCacheMiddleware(HttpCachePolicy): raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if (self.should_cache_request(request) - and self.should_cache_response(response)): - if self.use_dummy_cache: - if 'cached' not in response.flags: - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) - else: + if (self.policy.should_cache_request(request) + and self.policy.should_cache_response(response)): + if isinstance(self.policy, RFC2616Policy): if response.status != 304: self.storage.store_response(spider, request, response) self.stats.inc_value('httpcache/store', spider=spider) else: response.flags.append('cached') self.stats.inc_value('httpcache/hit', spider=spider) + else: + if 'cached' not in response.flags: + self.storage.store_response(spider, request, response) + self.stats.inc_value('httpcache/store', spider=spider) return response diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f494dc448..f7e04465f 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -141,7 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY = 'dummy' +HTTPCACHE_POLICY = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 7d505e4f9..a38775ab2 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -20,6 +20,9 @@ class HttpCacheMiddlewareTest(unittest.TestCase): storage_class = FilesystemCacheStorage realcache_storage_class = DbmRealCacheStorage + dummy_policy = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' + rfc2616_policy = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' + yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60) now = email.utils.formatdate() tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60) @@ -44,7 +47,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY': 'dummy' + 'HTTPCACHE_POLICY': self.dummy_policy } settings.update(new_settings) return Settings(settings) @@ -52,7 +55,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): @contextmanager def _storage(self, **new_settings): settings = self._get_settings(**new_settings) - if settings.get('HTTPCACHE_POLICY') == 'dummy': + if settings.get('HTTPCACHE_POLICY') == self.dummy_policy: storage = self.storage_class(settings) else: storage = self.realcache_storage_class(settings) @@ -182,10 +185,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags - def test_real_http_cache_middleware_response304_not_cached(self): + def test_middleware_rfc2616policy_response304_not_cached(self): # test response is not cached because the status is 304 Not Modified # (so it should be cached already) - with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: + with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', status=304) mw.process_response(self.request, response, self.spider) @@ -194,10 +197,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert mw.storage.retrieve_response(self.spider, self.request) is None assert mw.process_request(self.request, self.spider) is None - def test_real_http_cache_middleware_response_nostore_not_cached(self): + def test_middleware_rfc2616policy_response_nostore_not_cached(self): # test response is not cached because of the Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: + with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: assert mw.process_request(self.request, self.spider) is None response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, @@ -207,10 +210,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert mw.storage.retrieve_response(self.spider, self.request) is None assert mw.process_request(self.request, self.spider) is None - def test_real_http_cache_middleware_request_nostore_not_cached(self): + def test_middleware_rfc2616policy_request_nostore_not_cached(self): # test response is not cached because of the request's Cache-Control 'no-store' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: + with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) assert mw.process_request(request, self.spider) is None @@ -219,16 +222,16 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert mw.storage.retrieve_response(self.spider, request) is None assert mw.process_request(request, self.spider) is None - def test_real_http_cache_middleware_response_cached_and_fresh(self): + def test_middleware_rfc2616policy_response_cached_and_fresh(self): # test response cached and fresh - with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw: + with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: response = mw.process_response(self.request, self.response, self.spider) self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) assert 'cached' not in response.flags - def test_real_http_cache_middleware_response_cached_and_stale(self): + def test_middleware_rfc2616policy_response_cached_and_stale(self): # test response cached but stale - with self._middleware(HTTPCACHE_POLICY='rfc2616', + with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy, HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: response = Response('http://www.example.com', headers= {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, @@ -239,10 +242,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): response = mw.storage.retrieve_response(self.spider, self.request) assert isinstance(response, Request) - def test_real_http_cache_storage_response_cached_and_fresh(self): + def test_storage_rfc2616policy_response_cached_and_fresh(self): # test response is cached and is fresh # (response requested should be same as response received) - with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: + with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -252,10 +255,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): response2 = storage.retrieve_response(self.spider, self.request) self.assertEqualResponse(response, response2) - def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self): + def test_storage_rfc2616policy_response403_cached_and_further_requests_ignored(self): # test response is cached but further requests are ignored # because response status is 403 (as per the RFC) - with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: + with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -265,10 +268,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertRaises(IgnoreRequest, storage.retrieve_response, self.spider, self.request) - def test_real_http_cache_storage_response_cached_and_stale(self): + def test_storage_rfc2616policy_response_cached_and_stale(self): # test response is cached and is stale (no cache validators inserted) # (request should be same as response received) - with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: + with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -279,9 +282,9 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert isinstance(response2, Request) self.assertEqualRequest(self.request, response2) - def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self): + def test_storage_rfc2616policy_response_cached_and_stale_with_cache_validators(self): # test response is cached and is stale and cache validators are inserted - with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: + with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: assert storage.retrieve_response(self.spider, self.request) is None response = Response('http://www.example.com', headers= @@ -292,10 +295,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert isinstance(response2, Request) self.assertEqualRequestButWithCacheValidators(self.request, response2) - def test_real_http_cache_storage_response_cached_and_transparent(self): + def test_storage_rfc2616policy_response_cached_and_transparent(self): # test response is not cached because of the request's Cache-Control 'no-cache' directive # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._storage(HTTPCACHE_POLICY='rfc2616') as storage: + with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: request = Request('http://www.example.com', headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) assert storage.retrieve_response(self.spider, request) is None From cdecc760eeef2c7c60b2c9883c5c0eee0cd38e6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 4 Jan 2013 03:43:25 -0200 Subject: [PATCH 10/14] default httpcache to rfc2616 policy and improve storage and policy tests --- .../contrib/downloadermiddleware/httpcache.py | 320 ++++++++------ scrapy/contrib/httpcache.py | 237 ++++------ .../test_downloadermiddleware_httpcache.py | 410 +++++++++--------- 3 files changed, 494 insertions(+), 473 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index c267d1c8a..aa736708b 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -1,61 +1,167 @@ -import os -from os.path import join, exists from time import time -import cPickle as pickle - -from w3lib.http import headers_dict_to_raw, headers_raw_to_dict - +from email.utils import formatdate +from weakref import WeakKeyDictionary from scrapy import signals -from scrapy.http import Headers -from scrapy.http.request import Request from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.responsetypes import responsetypes -from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.project import data_path +from scrapy.contrib.httpcache import rfc1123_to_epoch, parse_cachecontrol class DummyPolicy(object): + def __init__(self, settings): self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - - def should_cache_response(self, response): - return response.status not in self.ignore_http_codes def should_cache_request(self, request): return urlparse_cached(request).scheme not in self.ignore_schemes + def should_cache_response(self, response, request): + return response.status not in self.ignore_http_codes + + def is_cached_response_fresh(self, response, request): + return True + + def is_cached_response_valid(self, cachedresponse, response, request): + return True + + +class RFC2616Policy(object): + + MAXAGE = 3600 * 24 * 365 # one year -class RFC2616Policy(DummyPolicy): def __init__(self, settings): - super(RFC2616Policy, self).__init__(settings) + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self._cc_parsed = WeakKeyDictionary() - def should_cache_response(self, response): - retval = super(RFC2616Policy, self).should_cache_response(response) - - if response.headers.has_key('cache-control'): - retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1) - #retval = retval and self.policy_response(response) - return retval + def _parse_cachecontrol(self, r): + if r not in self._cc_parsed: + cch = r.headers.get('Cache-Control', '') + self._cc_parsed[r] = parse_cachecontrol(cch) + return self._cc_parsed[r] def should_cache_request(self, request): - retval = super(RFC2616Policy, self).should_cache_request(request) - - if request.headers.has_key('cache-control'): - retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1) - #retval = retval and self.policy_request(request) - return retval + if urlparse_cached(request).scheme in self.ignore_schemes: + return False + cc = self._parse_cachecontrol(request) + # obey user-agent directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Any other is eligible for caching + return True + + def should_cache_response(self, response, request): + # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # Status code 206 is not included because cache can not deal with partial contents + cc = self._parse_cachecontrol(response) + # obey directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Never cache 304 (Not Modified) responses + elif response.status == 304: + return False + # Any hint on response expiration is good + elif 'max-age' in cc or 'Expires' in response.headers: + return True + # Firefox fallbacks this statuses to one year expiration if none is set + elif response.status in (300, 301, 308): + return True + # Other statuses without expiration requires at least one validator + elif response.status in (200, 203, 401): + return 'Last-Modified' in response.headers or 'ETag' in response.headers + # Any other is probably not eligible for caching + # Makes no sense to cache responses that does not contain expiration + # info and can not be revalidated + else: + return False + + def is_cached_response_fresh(self, cachedresponse, request): + cc = self._parse_cachecontrol(cachedresponse) + if 'no-cache' in cc: + return False + + now = time() + freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) + currentage = self._compute_current_age(cachedresponse, request, now) + if currentage < freshnesslifetime: + return True + # Cached response is stale, try to set validators if any + self._set_conditional_validators(request, cachedresponse) + return False + + def is_cached_response_valid(self, cachedresponse, response, request): + return response.status == 304 + + def _set_conditional_validators(self, request, cachedresponse): + if 'Last-Modified' in cachedresponse.headers: + request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + + if 'ETag' in cachedresponse.headers: + request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + + def _compute_freshness_lifetime(self, response, request, now): + # Reference nsHttpResponseHead::ComputeFresshnessLifetime + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 + cc = self._parse_cachecontrol(response) + if 'max-age' in cc: + try: + return max(0, int(cc['max-age'])) + except ValueError: + pass + + # Parse date header or synthesize it if none exists + date = rfc1123_to_epoch(response.headers.get('Date')) or now + + # Try HTTP/1.0 Expires header + if 'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers['Expires']) + # When parsing Expires header fails RFC 2616 section 14.21 says we + # should treat this as an expiration time in the past. + return max(0, expires - date) if expires else 0 + + # Fallback to heuristic using last-modified header + # This is not in RFC but on Firefox caching implementation + lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + if lastmodified and lastmodified <= date: + return (date - lastmodified) / 10 + + # This request can be cached indefinitely + if response.status in (300, 301, 308): + return self.MAXAGE + + # Insufficient information to compute fresshness lifetime + return 0 + + def _compute_current_age(self, response, request, now): + # Reference nsHttpResponseHead::ComputeCurrentAge + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html + currentage = 0 + # If Date header is not set we assume it is a fast connection, and + # clock is in sync with the server + date = rfc1123_to_epoch(response.headers.get('Date')) or now + if now > date: + currentage = now - date + + if 'Age' in response.headers: + try: + age = int(response.headers['Age']) + currentage = max(currentage, age) + except ValueError: + pass + + return currentage + class HttpCacheMiddleware(object): def __init__(self, settings, stats): if not settings.getbool('HTTPCACHE_ENABLED'): raise NotConfigured + self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') - self.policy = load_object(settings['HTTPCACHE_POLICY'])(settings) self.stats = stats @classmethod @@ -72,112 +178,72 @@ class HttpCacheMiddleware(object): self.storage.close_spider(spider) def process_request(self, request, spider): + # Skip uncacheable requests if not self.policy.should_cache_request(request): - return - response = self.storage.retrieve_response(spider, request) - - # Response cached, but stale - if response and type(response) is Request: - # Return None so that Scrapy continues processing - self.stats.inc_value('httpcache/revalidation', spider=spider) + request.meta['_dont_cache'] = True # flag as uncacheable return - if response and self.policy.should_cache_response(response): + # Look for cached response and check if expired + cachedresponse = self.storage.retrieve_response(spider, request) + if cachedresponse is None: + self.stats.inc_value('httpcache/miss', spider=spider) + if self.ignore_missing: + self.stats.inc_value('httpcache/ignore', spider=spider) + raise IgnoreRequest("Ignored request not in cache: %s" % request) + return # first time request + + # Return cached response only if not expired + cachedresponse.flags.append('cached') + if self.policy.is_cached_response_fresh(cachedresponse, request): self.stats.inc_value('httpcache/hit', spider=spider) - if isinstance(self.policy, RFC2616Policy): - # Response cached and fresh - raise IgnoreRequest("Ignored request already in cache: %s" % request) - else: - response.flags.append('cached') - return response + return cachedresponse - # Response not cached - self.stats.inc_value('httpcache/miss', spider=spider) - if self.ignore_missing: - raise IgnoreRequest("Ignored request not in cache: %s" % request) + # Keep a reference to cached response to avoid a second cache lookup on + # process_response hook + request.meta['cached_response'] = cachedresponse def process_response(self, request, response, spider): - if (self.policy.should_cache_request(request) - and self.policy.should_cache_response(response)): - if isinstance(self.policy, RFC2616Policy): - if response.status != 304: - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) - else: - response.flags.append('cached') - self.stats.inc_value('httpcache/hit', spider=spider) - else: - if 'cached' not in response.flags: - self.storage.store_response(spider, request, response) - self.stats.inc_value('httpcache/store', spider=spider) + # Skip cached responses and uncacheable requests + if 'cached' in response.flags or '_dont_cache' in request.meta: + request.meta.pop('_dont_cache', None) + return response + + # RFC2616 requires origin server to set Date header, + # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.18 + if 'Date' not in response.headers: + response.headers['Date'] = formatdate(usegmt=1) + + # Do not validate first-hand responses + cachedresponse = request.meta.pop('cached_response', None) + if cachedresponse is None: + self.stats.inc_value('httpcache/firsthand', spider=spider) + self._cache_response(spider, response, request, cachedresponse) + return response + + if self.policy.is_cached_response_valid(cachedresponse, response, request): + self.stats.inc_value('httpcache/revalidate', spider=spider) + return cachedresponse + + self.stats.inc_value('httpcache/invalidate', spider=spider) + self._cache_response(spider, response, request, cachedresponse) return response + def _cache_response(self, spider, response, request, cachedresponse): + if self.policy.should_cache_response(response, request): + self.stats.inc_value('httpcache/store', spider=spider) + self.storage.store_response(spider, request, response) + else: + self.stats.inc_value('httpcache/uncacheable', spider=spider) -class FilesystemCacheStorage(object): - def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) - self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') +from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage +class FilesystemCacheStorage(_FilesystemCacheStorage): - def open_spider(self, spider): - pass - - def close_spider(self, spider): - pass - - def retrieve_response(self, spider, request): - """Return response if present in cache, or None otherwise.""" - metadata = self._read_meta(spider, request) - if metadata is None: - return # not cached - rpath = self._get_request_path(spider, request) - with open(join(rpath, 'response_body'), 'rb') as f: - body = f.read() - with open(join(rpath, 'response_headers'), 'rb') as f: - rawheaders = f.read() - url = metadata.get('response_url') - status = metadata['status'] - headers = Headers(headers_raw_to_dict(rawheaders)) - respcls = responsetypes.from_args(headers=headers, url=url) - response = respcls(url=url, headers=headers, status=status, body=body) - return response - - def store_response(self, spider, request, response): - """Store the given response in the cache.""" - rpath = self._get_request_path(spider, request) - if not exists(rpath): - os.makedirs(rpath) - metadata = { - 'url': request.url, - 'method': request.method, - 'status': response.status, - 'response_url': response.url, - 'timestamp': time(), - } - with open(join(rpath, 'meta'), 'wb') as f: - f.write(repr(metadata)) - with open(join(rpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, protocol=2) - with open(join(rpath, 'response_headers'), 'wb') as f: - f.write(headers_dict_to_raw(response.headers)) - with open(join(rpath, 'response_body'), 'wb') as f: - f.write(response.body) - with open(join(rpath, 'request_headers'), 'wb') as f: - f.write(headers_dict_to_raw(request.headers)) - with open(join(rpath, 'request_body'), 'wb') as f: - f.write(request.body) - - def _get_request_path(self, spider, request): - key = request_fingerprint(request) - return join(self.cachedir, spider.name, key[0:2], key) - - def _read_meta(self, spider, request): - rpath = self._get_request_path(spider, request) - metapath = join(rpath, 'pickled_meta') - if not exists(metapath): - return # not found - mtime = os.stat(rpath).st_mtime - if 0 < self.expiration_secs < time() - mtime: - return # expired - with open(metapath, 'rb') as f: - return pickle.load(f) + def __init__(self, *args, **kwargs): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('Importing FilesystemCacheStorage from ' + 'scrapy.contrib.downloadermiddlware.httpcache is ' + 'deprecated, use scrapy.contrib.httpcache instead.', + category=ScrapyDeprecationWarning, stacklevel=1) + super(_FilesystemCacheStorage, self).__init__(*args, **kwargs) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index e815db3bc..d984ee8ed 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,12 +1,9 @@ import os -import calendar -import email.utils from time import time import cPickle as pickle - -from scrapy import log +from email.utils import mktime_tz, parsedate_tz +from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers -from scrapy.exceptions import IgnoreRequest from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path @@ -30,7 +27,7 @@ class DbmCacheStorage(object): def retrieve_response(self, spider, request): data = self._read_data(spider, request) if data is None: - return # not cached + return # not cached url = data['url'] status = data['status'] headers = Headers(data['headers']) @@ -55,149 +52,107 @@ class DbmCacheStorage(object): db = self.db tkey = '%s_time' % key if not db.has_key(tkey): - return # not found + return # not found ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): - return # expired + return # expired return pickle.loads(db['%s_data' % key]) def _request_key(self, request): return request_fingerprint(request) -class BaseRealCacheStorage(object): - """ - Most of the code was taken from the httplib2 (MIT License) - https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py - """ - - def parse_cache_control(self, headers): - retval = {} - if headers.has_key('cache-control'): - parts = headers['cache-control'].split(',') - parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")] - parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")] - retval = dict(parts_with_args + parts_wo_args) - return retval - - - def get(self, response_headers, request_headers): - """Determine freshness from the Date, Expires and Cache-Control headers. +class FilesystemCacheStorage(object): - We don't handle the following: - - 1. Cache-Control: max-stale - 2. Age: headers are not used in the calculations. - - Not that this algorithm is simpler than you might think - because we are operating as a private (non-shared) cache. - This lets us ignore 's-maxage'. We can also ignore - 'proxy-invalidate' since we aren't a proxy. - We will never return a stale document as - fresh as a design decision, and thus the non-implementation - of 'max-stale'. This also lets us safely ignore 'must-revalidate' - since we operate as if every server has sent 'must-revalidate'. - Since we are private we get to ignore both 'public' and - 'private' parameters. We also ignore 'no-transform' since - we don't do any transformations. - The 'no-store' parameter is handled at a higher level. - So the only Cache-Control parameters we look at are: - - no-cache - only-if-cached - max-age - min-fresh - """ - - retval = "STALE" - cc = self.parse_cache_control(request_headers) - cc_response = self.parse_cache_control(response_headers) - - if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1: - retval = "TRANSPARENT" - if 'cache-control' not in request_headers: - request_headers['cache-control'] = 'no-cache' - elif cc.has_key('no-cache'): - retval = "TRANSPARENT" - elif cc_response.has_key('no-cache'): - retval = "STALE" - elif cc.has_key('only-if-cached'): - retval = "FRESH" - elif response_headers.has_key('date'): - date = calendar.timegm(email.utils.parsedate_tz(response_headers['date'])) - now = time() - current_age = max(0, now - date) - if cc_response.has_key('max-age'): - try: - freshness_lifetime = int(cc_response['max-age']) - except ValueError: - freshness_lifetime = 0 - elif response_headers.has_key('expires'): - expires = email.utils.parsedate_tz(response_headers['expires']) - if None == expires: - freshness_lifetime = 0 - else: - freshness_lifetime = max(0, calendar.timegm(expires) - date) - else: - freshness_lifetime = 0 - if cc.has_key('max-age'): - try: - freshness_lifetime = int(cc['max-age']) - except ValueError: - freshness_lifetime = 0 - if cc.has_key('min-fresh'): - try: - min_fresh = int(cc['min-fresh']) - except ValueError: - min_fresh = 0 - current_age += min_fresh - if freshness_lifetime > current_age: - retval = "FRESH" - return retval - - def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''): - # Determine our course of action: - # Is the cached entry fresh or stale? - # - # There seems to be three possible answers: - # 1. [FRESH] Return the Response object - # 2. [STALE] Update the Request object with cache validators if available - # 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache) - entry_disposition = self.get(Headers(response_headers), Headers(request.headers)) - - # Per the RFC, requests should not be repeated in these situations - if response_status in [400, 401, 403, 410]: - raise IgnoreRequest("Ignored request because cached response status is %d." % response_status) - - if entry_disposition == "FRESH": - log.msg("Cache is FRESH", level=log.DEBUG, spider=spider) - - headers = Headers(response_headers) - respcls = responsetypes.from_args(headers=headers, url=response_url) - response = respcls(url=response_url, headers=headers, status=response_status, body=response_body) - - return response - - new_request = request.copy() - if entry_disposition == "STALE": - log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider) - if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers: - new_request.headers['If-None-Match'] = response_headers['ETag'] - if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers: - new_request.headers['If-Modified-Since'] = response_headers['Last-Modified'] - elif entry_disposition == "TRANSPARENT": - log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider) - - return new_request - - -class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage): def __init__(self, settings): - super(DbmRealCacheStorage, self).__init__(settings) - + self.cachedir = data_path(settings['HTTPCACHE_DIR']) + self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') + + def open_spider(self, spider): + pass + + def close_spider(self, spider): + pass + def retrieve_response(self, spider, request): - data = self._read_data(spider, request) - if data is None: - return # not cached - else: - return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body']) + """Return response if present in cache, or None otherwise.""" + metadata = self._read_meta(spider, request) + if metadata is None: + return # not cached + rpath = self._get_request_path(spider, request) + with open(os.path.join(rpath, 'response_body'), 'rb') as f: + body = f.read() + with open(os.path.join(rpath, 'response_headers'), 'rb') as f: + rawheaders = f.read() + url = metadata.get('response_url') + status = metadata['status'] + headers = Headers(headers_raw_to_dict(rawheaders)) + respcls = responsetypes.from_args(headers=headers, url=url) + response = respcls(url=url, headers=headers, status=status, body=body) + return response + + def store_response(self, spider, request, response): + """Store the given response in the cache.""" + rpath = self._get_request_path(spider, request) + if not os.path.exists(rpath): + os.makedirs(rpath) + metadata = { + 'url': request.url, + 'method': request.method, + 'status': response.status, + 'response_url': response.url, + 'timestamp': time(), + } + with open(os.path.join(rpath, 'meta'), 'wb') as f: + f.write(repr(metadata)) + with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: + pickle.dump(metadata, f, protocol=2) + with open(os.path.join(rpath, 'response_headers'), 'wb') as f: + f.write(headers_dict_to_raw(response.headers)) + with open(os.path.join(rpath, 'response_body'), 'wb') as f: + f.write(response.body) + with open(os.path.join(rpath, 'request_headers'), 'wb') as f: + f.write(headers_dict_to_raw(request.headers)) + with open(os.path.join(rpath, 'request_body'), 'wb') as f: + f.write(request.body) + + def _get_request_path(self, spider, request): + key = request_fingerprint(request) + return os.path.join(self.cachedir, spider.name, key[0:2], key) + + def _read_meta(self, spider, request): + rpath = self._get_request_path(spider, request) + metapath = os.path.join(rpath, 'pickled_meta') + if not os.path.exists(metapath): + return # not found + mtime = os.stat(rpath).st_mtime + if 0 < self.expiration_secs < time() - mtime: + return # expired + with open(metapath, 'rb') as f: + return pickle.load(f) + + +def parse_cachecontrol(header): + """Parse Cache-Control header + + http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + + >>> cachecontrol_directives('public, max-age=3600') + {'public': None, 'max-age': '3600'} + >>> cachecontrol_directives('') + {} + + """ + directives = {} + for directive in header.split(','): + key, sep, val = directive.strip().partition('=') + if key: + directives[key.lower()] = val if sep else None + return directives + + +def rfc1123_to_epoch(date_str): + try: + return mktime_tz(parsedate_tz(date_str)) + except Exception: + return None diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index a38775ab2..28b3026f0 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -10,31 +10,27 @@ from scrapy.spider import BaseSpider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler -from scrapy.contrib.downloadermiddleware.httpcache import \ - FilesystemCacheStorage, HttpCacheMiddleware -from scrapy.contrib.httpcache import DbmRealCacheStorage +from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware -class HttpCacheMiddlewareTest(unittest.TestCase): +class _BaseTest(unittest.TestCase): - storage_class = FilesystemCacheStorage - realcache_storage_class = DbmRealCacheStorage - - dummy_policy = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - rfc2616_policy = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' - - yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60) - now = email.utils.formatdate() - tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60) + storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' def setUp(self): + self.yesterday = email.utils.formatdate(time.time() - 86400) + self.today = email.utils.formatdate() + self.tomorrow = email.utils.formatdate(time.time() + 86400) self.crawler = get_crawler() self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', - headers={'User-Agent': 'test'}) - self.response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html'}, body='test body', status=202) + headers={'User-Agent': 'test'}) + self.response = Response('http://www.example.com', + headers={'Content-Type': 'text/html'}, + body='test body', + status=202) self.crawler.stats.open_spider(self.spider) def tearDown(self): @@ -47,23 +43,21 @@ class HttpCacheMiddlewareTest(unittest.TestCase): 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], - 'HTTPCACHE_POLICY': self.dummy_policy + 'HTTPCACHE_POLICY': self.policy_class, + 'HTTPCACHE_STORAGE': self.storage_class, } settings.update(new_settings) return Settings(settings) @contextmanager def _storage(self, **new_settings): - settings = self._get_settings(**new_settings) - if settings.get('HTTPCACHE_POLICY') == self.dummy_policy: - storage = self.storage_class(settings) - else: - storage = self.realcache_storage_class(settings) - storage.open_spider(self.spider) - try: - yield storage - finally: - storage.close_spider(self.spider) + with self._middleware(**new_settings) as mw: + yield mw.storage + + @contextmanager + def _policy(self, **new_settings): + with self._middleware(**new_settings) as mw: + yield mw.policy @contextmanager def _middleware(self, **new_settings): @@ -75,6 +69,27 @@ class HttpCacheMiddlewareTest(unittest.TestCase): finally: mw.spider_closed(self.spider) + def assertEqualResponse(self, response1, response2): + self.assertEqual(response1.url, response2.url) + self.assertEqual(response1.status, response2.status) + self.assertEqual(response1.headers, response2.headers) + self.assertEqual(response1.body, response2.body) + + def assertEqualRequest(self, request1, request2): + self.assertEqual(request1.url, request2.url) + self.assertEqual(request1.headers, request2.headers) + self.assertEqual(request1.body, request2.body) + + def assertEqualRequestButWithCacheValidators(self, request1, request2): + self.assertEqual(request1.url, request2.url) + assert not 'If-None-Match' in request1.headers + assert not 'If-Modified-Since' in request1.headers + assert any(h in request2.headers for h in ('If-None-Match', 'If-Modified-Since')) + self.assertEqual(request1.body, request2.body) + + +class DefaultStorageTest(_BaseTest): + def test_storage(self): with self._storage() as storage: request2 = self.request.copy() @@ -95,11 +110,23 @@ class HttpCacheMiddlewareTest(unittest.TestCase): time.sleep(0.5) # give the chance to expire assert storage.retrieve_response(self.spider, self.request) + +class DbmStorageTest(DefaultStorageTest): + + storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' + + +class FilesystemStorageTest(DefaultStorageTest): + + storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' + + +class DefaultMiddlewaretest(_BaseTest): + def test_middleware(self): with self._middleware() as mw: assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) - response = mw.process_request(self.request, self.spider) assert isinstance(response, HtmlResponse) self.assertEqualResponse(self.response, response) @@ -109,10 +136,8 @@ class HttpCacheMiddlewareTest(unittest.TestCase): with self._middleware() as mw: req = Request('http://host.com/path') res = Response('http://host2.net/test.html') - assert mw.process_request(req, self.spider) is None mw.process_response(req, res, self.spider) - cached = mw.process_request(req, self.spider) assert isinstance(cached, Response) self.assertEqualResponse(res, cached) @@ -185,192 +210,167 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqualResponse(self.response, response) assert 'cached' in response.flags - def test_middleware_rfc2616policy_response304_not_cached(self): - # test response is not cached because the status is 304 Not Modified - # (so it should be cached already) - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - assert mw.process_request(self.request, self.spider) is None - response = Response('http://www.example.com', status=304) - mw.process_response(self.request, response, self.spider) - assert 'cached' in response.flags - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None +class DummyMiddlewareTest(DefaultStorageTest): - def test_middleware_rfc2616policy_response_nostore_not_cached(self): - # test response is not cached because of the Cache-Control 'no-store' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - assert mw.process_request(self.request, self.spider) is None - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Cache-Control': 'no-store'}, - body='test body', status=200) - mw.process_response(self.request, response, self.spider) + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - assert mw.storage.retrieve_response(self.spider, self.request) is None - assert mw.process_request(self.request, self.spider) is None - def test_middleware_rfc2616policy_request_nostore_not_cached(self): - # test response is not cached because of the request's Cache-Control 'no-store' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - request = Request('http://www.example.com', - headers={'User-Agent': 'test', 'Cache-Control': 'no-store'}) - assert mw.process_request(request, self.spider) is None - mw.process_response(request, self.response, self.spider) +class RFC2616MiddlewareTest(DefaultStorageTest): - assert mw.storage.retrieve_response(self.spider, request) is None - assert mw.process_request(request, self.spider) is None + policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' - def test_middleware_rfc2616policy_response_cached_and_fresh(self): - # test response cached and fresh - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy) as mw: - response = mw.process_response(self.request, self.response, self.spider) - self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider) - assert 'cached' not in response.flags + def _process_requestresponse(self, mw, request, response): + try: + result = mw.process_request(request, self.spider) + if result: + assert isinstance(result, (Request, Response)) + return result + else: + result = mw.process_response(request, response, self.spider) + assert isinstance(result, Response) + return result + except Exception: + print 'Request', request + print 'Response', response + print 'Result', result + raise - def test_middleware_rfc2616policy_response_cached_and_stale(self): - # test response cached but stale - with self._middleware(HTTPCACHE_POLICY=self.rfc2616_policy, - HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw: - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Cache-Control': 'no-cache'}, - body='test body', status=200) - mw.process_response(self.request, response, self.spider) - assert mw.process_request(self.request, self.spider) is None + def test_request_cacheability(self): + res0 = Response(self.request.url, status=200, + headers={'Expires': self.tomorrow}) + req0 = Request('http://example.com') + req1 = req0.replace(headers={'Cache-Control': 'no-store'}) + with self._middleware() as mw: + res1 = self._process_requestresponse(mw, req1, res0) + self.assertEqualResponse(res1, res0) + assert mw.storage.retrieve_response(self.spider, req1) is None + res2 = self._process_requestresponse(mw, req0, res0) + assert 'cached' not in res2.flags + res3 = mw.process_request(req0, self.spider) + assert 'cached' in res3.flags + self.assertEqualResponse(res2, res3) - response = mw.storage.retrieve_response(self.spider, self.request) - assert isinstance(response, Request) + def test_response_cacheability(self): + responses = [ + # 304 is not cacheable no matter what servers sends + (False, 304, {}), + (False, 304, {'Last-Modified': self.yesterday}), + (False, 304, {'Expires': self.tomorrow}), + (False, 304, {'Etag': 'bar'}), + (False, 304, {'Cache-Control': 'max-age=3600'}), + # Always obey no-store cache control + (False, 200, {'Cache-Control': 'no-store'}), + (False, 200, {'Cache-Control': 'no-store, max-age=300'}), # invalid + (False, 200, {'Cache-Control': 'no-store', 'Expires': self.tomorrow}), # invalid + # Ignore responses missing expiration and/or validation headers + (False, 200, {}), + (False, 302, {}), + (False, 307, {}), + (False, 404, {}), + # Cache responses with expiration and/or validation headers + (True, 200, {'Last-Modified': self.yesterday}), + (True, 203, {'Last-Modified': self.yesterday}), + (True, 300, {'Last-Modified': self.yesterday}), + (True, 301, {'Last-Modified': self.yesterday}), + (True, 401, {'Last-Modified': self.yesterday}), + (True, 404, {'Cache-Control': 'public, max-age=600'}), + (True, 302, {'Expires': self.tomorrow}), + (True, 200, {'Etag': 'foo'}), + ] + with self._middleware() as mw: + for idx, (shouldcache, status, headers) in enumerate(responses): + req0 = Request('http://example-%d.com' % idx) + res0 = Response(req0.url, status=status, headers=headers) + res1 = self._process_requestresponse(mw, req0, res0) + res304 = res0.replace(status=304) + res2 = self._process_requestresponse(mw, req0, res304 if shouldcache else res0) + self.assertEqualResponse(res1, res0) + self.assertEqualResponse(res2, res0) + resc = mw.storage.retrieve_response(self.spider, req0) + if shouldcache: + self.assertEqualResponse(resc, res1) + assert 'cached' in res2.flags and res2.status != 304 + else: + self.assertFalse(resc) + assert 'cached' not in res2.flags - def test_storage_rfc2616policy_response_cached_and_fresh(self): - # test response is cached and is fresh - # (response requested should be same as response received) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None + def test_cached_and_fresh(self): + sampledata = [ + (200, {'Date': self.yesterday, 'Expires': self.tomorrow}), + (200, {'Date': self.yesterday, 'Cache-Control': 'max-age=86405'}), + (200, {'Age': '299', 'Cache-Control': 'max-age=300'}), + # Obey max-age if present over any others + (200, {'Date': self.today, + 'Age': '86405', + 'Cache-Control': 'max-age=' + str(86400 * 3), + 'Expires': self.yesterday, + 'Last-Modified': self.yesterday, + }), + # obey Expires if max-age is not present + (200, {'Date': self.yesterday, + 'Age': '86400', + 'Cache-Control': 'public', + 'Expires': self.tomorrow, + 'Last-Modified': self.yesterday, + }), + # Default missing Date header to right now + (200, {'Expires': self.tomorrow}), + # Firefox - Expires if age is greater than 10% of (Date - Last-Modified) + (200, {'Date': self.today, 'Last-Modified': self.yesterday, 'Age': str(86400 / 10 - 1)}), + # Firefox - Set one year maxage to permanent redirects missing expiration info + (300, {}), (301, {}), (308, {}), + ] + with self._middleware() as mw: + for idx, (status, headers) in enumerate(sampledata): + req0 = Request('http://example-%d.com' % idx) + res0 = Response(req0.url, status=status, headers=headers) + # cache fresh response + res1 = self._process_requestresponse(mw, req0, res0) + self.assertEqualResponse(res1, res0) + assert 'cached' not in res1.flags + # return fresh cached response without network interaction + res2 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res1, res2) + assert 'cached' in res2.flags - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, - body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - self.assertEqualResponse(response, response2) + def test_cached_and_stale(self): + sampledata = [ + (200, {'Date': self.today, 'Expires': self.yesterday}), + (200, {'Date': self.today, 'Expires': self.yesterday, 'Last-Modified': self.yesterday}), + (200, {'Expires': self.yesterday}), + (200, {'Expires': self.yesterday, 'ETag': 'foo'}), + (200, {'Expires': self.yesterday, 'Last-Modified': self.yesterday}), + (200, {'Expires': self.tomorrow, 'Age': '86405'}), + (200, {'Cache-Control': 'max-age=86400', 'Age': '86405'}), + # no-cache forces expiration, also revalidation if validators exists + (200, {'Cache-Control': 'no-cache'}), + (200, {'Cache-Control': 'no-cache', 'ETag': 'foo'}), + (200, {'Cache-Control': 'no-cache', 'Last-Modified': self.yesterday}), + ] + with self._middleware() as mw: + for idx, (status, headers) in enumerate(sampledata): + req0 = Request('http://example-%d.com' % idx) + res0a = Response(req0.url, status=status, headers=headers) + # cache expired response + res1 = self._process_requestresponse(mw, req0, res0a) + self.assertEqualResponse(res1, res0a) + assert 'cached' not in res1.flags + # Same request but as cached response is stale a new response must + # be returned + res0b = res0a.replace(body='bar') + res2 = self._process_requestresponse(mw, req0, res0b) + self.assertEqualResponse(res2, res0b) + assert 'cached' not in res2.flags + # Previous response expired too, subsequent request to same + # resource must revalidate and succeed on 304 if validators + # are present + if 'ETag' in headers or 'Last-Modified' in headers: + res0c = res0b.replace(status=304) + res3 = self._process_requestresponse(mw, req0, res0c) + self.assertEqualResponse(res3, res0b) + assert 'cached' in res3.flags - def test_storage_rfc2616policy_response403_cached_and_further_requests_ignored(self): - # test response is cached but further requests are ignored - # because response status is 403 (as per the RFC) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow}, - body='test body', status=403) - storage.store_response(self.spider, self.request, response) - self.assertRaises(IgnoreRequest, storage.retrieve_response, - self.spider, self.request) - - def test_storage_rfc2616policy_response_cached_and_stale(self): - # test response is cached and is stale (no cache validators inserted) - # (request should be same as response received) - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday}, - body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - assert isinstance(response2, Request) - self.assertEqualRequest(self.request, response2) - - def test_storage_rfc2616policy_response_cached_and_stale_with_cache_validators(self): - # test response is cached and is stale and cache validators are inserted - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - assert storage.retrieve_response(self.spider, self.request) is None - - response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday, - 'Last-Modified': self.yesterday}, body='test body', status=200) - storage.store_response(self.spider, self.request, response) - response2 = storage.retrieve_response(self.spider, self.request) - assert isinstance(response2, Request) - self.assertEqualRequestButWithCacheValidators(self.request, response2) - - def test_storage_rfc2616policy_response_cached_and_transparent(self): - # test response is not cached because of the request's Cache-Control 'no-cache' directive - # http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2 - with self._storage(HTTPCACHE_POLICY=self.rfc2616_policy) as storage: - request = Request('http://www.example.com', - headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'}) - assert storage.retrieve_response(self.spider, request) is None - storage.store_response(self.spider, request, self.response) - response = storage.retrieve_response(self.spider, request) - assert isinstance(response, Request) - self.assertEqualRequest(request, response) - - def test_http_cache_request_policy(self): - callback = lambda r: (r.url == 'http://www.example.com/') - - # Should be cached - req, res = Request('http://www.example.com/'), Response('http://www.example.com/') - with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - cached = mw.process_request(req, self.spider) - assert isinstance(cached, Response), type(cached) - self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags - - # Should not be cached - req, res = Request('http://www.test.com/'), Response('http://www.test.com/') - with self._middleware(HTTPCACHE_POLICY_REQUEST=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None - - def test_http_cache_response_policy(self): - callback = lambda r: (r.url == 'http://www.example.com/') - - # Should be cached - req, res = Request('http://www.example.com/'), Response('http://www.example.com/') - with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - cached = mw.process_request(req, self.spider) - assert isinstance(cached, Response), type(cached) - self.assertEqualResponse(res, cached) - assert 'cached' in cached.flags - - # Should not be cached - req, res = Request('http://www.test.com/'), Response('http://www.test.com/') - with self._middleware(HTTPCACHE_POLICY_RESPONSE=callback) as mw: - assert mw.process_request(req, self.spider) is None - mw.process_response(req, res, self.spider) - - assert mw.storage.retrieve_response(self.spider, req) is None - assert mw.process_request(req, self.spider) is None - - def assertEqualResponse(self, response1, response2): - self.assertEqual(response1.url, response2.url) - self.assertEqual(response1.status, response2.status) - self.assertEqual(response1.headers, response2.headers) - self.assertEqual(response1.body, response2.body) - - def assertEqualRequest(self, request1, request2): - self.assertEqual(request1.url, request2.url) - self.assertEqual(request1.headers, request2.headers) - self.assertEqual(request1.body, request2.body) - - def assertEqualRequestButWithCacheValidators(self, request1, request2): - self.assertEqual(request1.url, request2.url) - assert not request1.headers.has_key('If-None-Match') - assert not request1.headers.has_key('If-Modified-Since') - assert (request2.headers.has_key('If-None-Match') or \ - request2.headers.has_key('If-Modified-Since')) - self.assertEqual(request1.body, request2.body) if __name__ == '__main__': unittest.main() From 3f03a2ca509b7e8961ced2f5f98bf162077bf03f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Fri, 4 Jan 2013 04:20:45 -0200 Subject: [PATCH 11/14] requests with no-cache set must force revalidation of cached responses --- scrapy/contrib/downloadermiddleware/httpcache.py | 3 ++- scrapy/tests/test_downloadermiddleware_httpcache.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index aa736708b..b3ac193b4 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -79,7 +79,8 @@ class RFC2616Policy(object): def is_cached_response_fresh(self, cachedresponse, request): cc = self._parse_cachecontrol(cachedresponse) - if 'no-cache' in cc: + ccreq = self._parse_cachecontrol(request) + if 'no-cache' in cc or 'no-cache' in ccreq: return False now = time() diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 28b3026f0..00baf4748 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -241,15 +241,27 @@ class RFC2616MiddlewareTest(DefaultStorageTest): headers={'Expires': self.tomorrow}) req0 = Request('http://example.com') req1 = req0.replace(headers={'Cache-Control': 'no-store'}) + req2 = req0.replace(headers={'Cache-Control': 'no-cache'}) with self._middleware() as mw: + # response for a request with no-store must not be cached res1 = self._process_requestresponse(mw, req1, res0) self.assertEqualResponse(res1, res0) assert mw.storage.retrieve_response(self.spider, req1) is None + # Re-do request without no-store and expect it to be cached res2 = self._process_requestresponse(mw, req0, res0) assert 'cached' not in res2.flags res3 = mw.process_request(req0, self.spider) assert 'cached' in res3.flags self.assertEqualResponse(res2, res3) + # request with no-cache directive must not return cached response + # but it allows new response to be stored + res0b = res0.replace(body='foo') + res4 = self._process_requestresponse(mw, req2, res0b) + self.assertEqualResponse(res4, res0b) + assert 'cached' not in res4.flags + res5 = self._process_requestresponse(mw, req0, None) + self.assertEqualResponse(res5, res0b) + assert 'cached' in res5.flags def test_response_cacheability(self): responses = [ From 5d3a4d755fa7dd59be1559cf18ac8ecdbae21bdd Mon Sep 17 00:00:00 2001 From: Pedro Faustino Date: Sun, 6 Jan 2013 18:53:14 +0000 Subject: [PATCH 13/14] Update downloader middleware documentation --- docs/topics/downloader-middleware.rst | 98 +++++++++------------------ 1 file changed, 32 insertions(+), 66 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 48292e10c..a47ffa83e 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -293,44 +293,46 @@ HttpCacheMiddleware .. class:: HttpCacheMiddleware - There are two types of caches: + This middleware provides low-level cache to all HTTP requests and responses. + It has to be combined with a cache storage backend as well as a cache policy. - * Dummy cache + Scrapy ships with two HTTP cache storage backends: - This middleware was designed as a dummy low-level cache to all HTTP - requests and responses, with no awareness of any HTTP Cache-Control + * :ref:`httpcache-dbm-backend` + * :ref:`httpcache-fs-backend` + + You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` + setting. Or you can also implement your own storage backend. + + + Scrapy ships with two HTTP cache policies: + + * Dummy policy + + This policy has no awareness of any HTTP Cache-Control directives. Every request and its corresponding response are cached. When the same request is seen again, the response is returned without transferring anything from the Internet. - The HTTP cache is useful for testing spiders faster (without having to + The Dummy policy is useful for testing spiders faster (without having to wait for downloads every time) and for trying your spider offline, when an Internet connection is not available. The goal is to be able to - "replay" a spider run *exactly as it run before* and not to use HTTP - caching (to save bandwidth and speed up the crawl). + "replay" a spider run *exactly as it ran before*. - Scrapy ships with two storage backends for the dummy HTTP cache middleware: - - * :ref:`httpcache-dbm-backend` - * :ref:`httpcache-fs-backend` + This is the default cache policy. - * Real HTTP cache + * RFC2616 policy - This middleware was designed as a real HTTP cache with HTTP Cache-Control - awareness, aimed at production and used in continuous runs to avoid - downloading unmodified data (to save bandwidth and speed up crawls). + This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP + Cache-Control awareness, aimed at production and used in continuous + runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). - In order to use the real HTTP cache, set: + In order to use this policy, set: - * :setting:`HTTPCACHE_USE_DUMMY` to ``False`` - * :setting:`HTTPCACHE_STORAGE` to ``'scrapy.contrib.httpcache.DbmRealCacheStorage'`` + * :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy`` - Scrapy ships with one storage backend for the real HTTP cache middleware: - - * :ref:`httprealcache-dbm-backend` - - You can change the storage backend with the :setting:`HTTPCACHE_STORAGE` - setting. Or you can also implement your own backend. + You can change the HTTP cache policy with the :setting:`HTTPCACHE_POLICY` + setting. Or you can also implement your own policy. .. _httpcache-dbm-backend: @@ -346,20 +348,6 @@ to ``scrapy.contrib.httpcache.DbmCacheStorage``. By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. -.. _httprealcache-dbm-backend: - -DBM storage backend (real HTTP cache) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It inherits from both ``scrapy.contrib.httpcache.DbmCacheStorage`` and -``scrapy.contrib.httpcache.BaseRealCacheStorage``, the latter providing -HTTP Cache-Control awareness. To use it set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.httpcache.DbmRealCacheStorage`` and -:setting:`HTTPCACHE_USE_DUMMY` to ``False``. - -If you need to create your own storage backend with HTTP Cache-Control awareness, -you can inherit from ``scrapy.contrib.httpcache.BaseRealCacheStorage``. - .. _httpcache-fs-backend: File system backend @@ -408,15 +396,6 @@ Whether the HTTP cache will be enabled. .. versionchanged:: 0.11 Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. -.. setting:: HTTPCACHE_USE_DUMMY - -HTTPCACHE_USE_DUMMY -^^^^^^^^^^^^^^^^^^^ - -Default: ``True`` - -Whether to use the dummy or the real HTTP cache. The default is set to ``True`` for backwards compatibility. - .. setting:: HTTPCACHE_EXPIRATION_SECS HTTPCACHE_EXPIRATION_SECS @@ -495,29 +474,16 @@ Default: ``'anydbm'`` The database module to use in the :ref:`DBM storage backend `. This setting is specific to the DBM backend. -HTTPCACHE_POLICY_REQUEST -^^^^^^^^^^^^^^^^^^^^^^^^ +.. setting:: HTTPCACHE_POLICY + +HTTPCACHE_POLICY +^^^^^^^^^^^^^^^^ .. versionadded:: 0.18 -Default: ```lambda request: True``` +Default: ``'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'`` -A callback function used by the HTTP cache to decide whether a request -is cacheable. The function should take a :class:`~scrapy.http.Request` -object as a parameter and return ``True`` if a cached response can be returned; -or ``False`` if it should be fetched again. - -HTTPCACHE_POLICY_RESPONSE -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. versionadded:: 0.18 - -Default: ```lambda response: True``` - -A callback function used by the HTTP cache to decide whether a response -is cacheable. The function should take a :class:`~scrapy.http.Response` -object as a parameter and return ``True`` if a response can be cached; -or ``False`` if it should not be stored in the cache. +The class which implements the cache policy. HttpCompressionMiddleware ------------------------- From 864a7aef87172ea3ff182ea56caa95e6912652d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 8 Jan 2013 17:26:32 -0200 Subject: [PATCH 14/14] More httpcache updates * Change default cache policy to RFC2616 * Update HttpCacheMiddleware documentation * Move policies to scrapy.contrib.httpcache * remove a lint error for .has_key() usage in DBM storage backend --- docs/topics/downloader-middleware.rst | 97 ++++++----- .../contrib/downloadermiddleware/httpcache.py | 147 ----------------- scrapy/contrib/httpcache.py | 155 +++++++++++++++++- scrapy/settings/default_settings.py | 4 +- .../test_downloadermiddleware_httpcache.py | 15 +- 5 files changed, 221 insertions(+), 197 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a47ffa83e..395a7a817 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -285,6 +285,7 @@ HttpAuthMiddleware .. _Basic access authentication: http://en.wikipedia.org/wiki/Basic_access_authentication + HttpCacheMiddleware ------------------- @@ -298,64 +299,84 @@ HttpCacheMiddleware Scrapy ships with two HTTP cache storage backends: - * :ref:`httpcache-dbm-backend` - * :ref:`httpcache-fs-backend` + * :ref:`httpcache-storage-dbm` + * :ref:`httpcache-storage-fs` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own storage backend. - - + Scrapy ships with two HTTP cache policies: - * Dummy policy - - This policy has no awareness of any HTTP Cache-Control - directives. Every request and its corresponding response are cached. - When the same request is seen again, the response is returned - without transferring anything from the Internet. - - The Dummy policy is useful for testing spiders faster (without having to - wait for downloads every time) and for trying your spider offline, when - an Internet connection is not available. The goal is to be able to - "replay" a spider run *exactly as it ran before*. - - This is the default cache policy. - - * RFC2616 policy - - This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP - Cache-Control awareness, aimed at production and used in continuous - runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). - - In order to use this policy, set: - - * :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy`` + * :ref:`httpcache-policy-rfc2616` + * :ref:`httpcache-policy-dummy` You can change the HTTP cache policy with the :setting:`HTTPCACHE_POLICY` setting. Or you can also implement your own policy. -.. _httpcache-dbm-backend: + +.. _httpcache-policy-rfc2616: + +RFC2616 policy (default) +~~~~~~~~~~~~~~~~~~~~~~~~ + +This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP +Cache-Control awareness, aimed at production and used in continuous +runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). + +In order to use this policy, set: + +* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.RFC2616Policy`` + +This is the default cache policy. + + +.. _httpcache-policy-dummy: + +Dummy policy +~~~~~~~~~~~~ + +This policy has no awareness of any HTTP Cache-Control directives. +Every request and its corresponding response are cached. When the same +request is seen again, the response is returned without transferring +anything from the Internet. + +The Dummy policy is useful for testing spiders faster (without having +to wait for downloads every time) and for trying your spider offline, +when an Internet connection is not available. The goal is to be able to +"replay" a spider run *exactly as it ran before*. + +In order to use this policy, set: + +* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.DummyPolicy`` + + +.. _httpcache-storage-dbm: DBM storage backend (default) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 0.13 -A DBM_ storage backend is available for the HTTP cache middleware. To use it -(note: it is the default storage backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.httpcache.DbmCacheStorage``. +A DBM_ storage backend is available for the HTTP cache middleware. By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. -.. _httpcache-fs-backend: +In order to use this storage backend, set: -File system backend -~~~~~~~~~~~~~~~~~~~ +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.DbmCacheStorage`` + + +.. _httpcache-storage-fs: + +Filesystem storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~ A file system storage backend is also available for the HTTP cache middleware. -To use it (instead of the default DBM_ storage backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage``. + +In order to use this storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.FilesystemCacheStorage`` Each request/response pair is stored in a different directory containing the following files: @@ -376,6 +397,7 @@ inefficient in many file systems). An example directory could be:: /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 + HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -481,10 +503,11 @@ HTTPCACHE_POLICY .. versionadded:: 0.18 -Default: ``'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'`` +Default: ``'scrapy.contrib.httpcache.RFC2616Policy'`` The class which implements the cache policy. + HttpCompressionMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b3ac193b4..ebfd14c1c 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -8,153 +8,6 @@ from scrapy.utils.misc import load_object from scrapy.contrib.httpcache import rfc1123_to_epoch, parse_cachecontrol -class DummyPolicy(object): - - def __init__(self, settings): - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - - def should_cache_request(self, request): - return urlparse_cached(request).scheme not in self.ignore_schemes - - def should_cache_response(self, response, request): - return response.status not in self.ignore_http_codes - - def is_cached_response_fresh(self, response, request): - return True - - def is_cached_response_valid(self, cachedresponse, response, request): - return True - - -class RFC2616Policy(object): - - MAXAGE = 3600 * 24 * 365 # one year - - def __init__(self, settings): - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self._cc_parsed = WeakKeyDictionary() - - def _parse_cachecontrol(self, r): - if r not in self._cc_parsed: - cch = r.headers.get('Cache-Control', '') - self._cc_parsed[r] = parse_cachecontrol(cch) - return self._cc_parsed[r] - - def should_cache_request(self, request): - if urlparse_cached(request).scheme in self.ignore_schemes: - return False - cc = self._parse_cachecontrol(request) - # obey user-agent directive "Cache-Control: no-store" - if 'no-store' in cc: - return False - # Any other is eligible for caching - return True - - def should_cache_response(self, response, request): - # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 - # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 - # Status code 206 is not included because cache can not deal with partial contents - cc = self._parse_cachecontrol(response) - # obey directive "Cache-Control: no-store" - if 'no-store' in cc: - return False - # Never cache 304 (Not Modified) responses - elif response.status == 304: - return False - # Any hint on response expiration is good - elif 'max-age' in cc or 'Expires' in response.headers: - return True - # Firefox fallbacks this statuses to one year expiration if none is set - elif response.status in (300, 301, 308): - return True - # Other statuses without expiration requires at least one validator - elif response.status in (200, 203, 401): - return 'Last-Modified' in response.headers or 'ETag' in response.headers - # Any other is probably not eligible for caching - # Makes no sense to cache responses that does not contain expiration - # info and can not be revalidated - else: - return False - - def is_cached_response_fresh(self, cachedresponse, request): - cc = self._parse_cachecontrol(cachedresponse) - ccreq = self._parse_cachecontrol(request) - if 'no-cache' in cc or 'no-cache' in ccreq: - return False - - now = time() - freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) - currentage = self._compute_current_age(cachedresponse, request, now) - if currentage < freshnesslifetime: - return True - # Cached response is stale, try to set validators if any - self._set_conditional_validators(request, cachedresponse) - return False - - def is_cached_response_valid(self, cachedresponse, response, request): - return response.status == 304 - - def _set_conditional_validators(self, request, cachedresponse): - if 'Last-Modified' in cachedresponse.headers: - request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] - - if 'ETag' in cachedresponse.headers: - request.headers['If-None-Match'] = cachedresponse.headers['ETag'] - - def _compute_freshness_lifetime(self, response, request, now): - # Reference nsHttpResponseHead::ComputeFresshnessLifetime - # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 - cc = self._parse_cachecontrol(response) - if 'max-age' in cc: - try: - return max(0, int(cc['max-age'])) - except ValueError: - pass - - # Parse date header or synthesize it if none exists - date = rfc1123_to_epoch(response.headers.get('Date')) or now - - # Try HTTP/1.0 Expires header - if 'Expires' in response.headers: - expires = rfc1123_to_epoch(response.headers['Expires']) - # When parsing Expires header fails RFC 2616 section 14.21 says we - # should treat this as an expiration time in the past. - return max(0, expires - date) if expires else 0 - - # Fallback to heuristic using last-modified header - # This is not in RFC but on Firefox caching implementation - lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) - if lastmodified and lastmodified <= date: - return (date - lastmodified) / 10 - - # This request can be cached indefinitely - if response.status in (300, 301, 308): - return self.MAXAGE - - # Insufficient information to compute fresshness lifetime - return 0 - - def _compute_current_age(self, response, request, now): - # Reference nsHttpResponseHead::ComputeCurrentAge - # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html - currentage = 0 - # If Date header is not set we assume it is a fast connection, and - # clock is in sync with the server - date = rfc1123_to_epoch(response.headers.get('Date')) or now - if now > date: - currentage = now - date - - if 'Age' in response.headers: - try: - age = int(response.headers['Age']) - currentage = max(currentage, age) - except ValueError: - pass - - return currentage - - class HttpCacheMiddleware(object): def __init__(self, settings, stats): diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index d984ee8ed..0891c6355 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,12 +1,161 @@ import os -from time import time import cPickle as pickle +from time import time +from weakref import WeakKeyDictionary from email.utils import mktime_tz, parsedate_tz from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path +from scrapy.utils.httpobj import urlparse_cached + + +class DummyPolicy(object): + + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) + + def should_cache_request(self, request): + return urlparse_cached(request).scheme not in self.ignore_schemes + + def should_cache_response(self, response, request): + return response.status not in self.ignore_http_codes + + def is_cached_response_fresh(self, response, request): + return True + + def is_cached_response_valid(self, cachedresponse, response, request): + return True + + +class RFC2616Policy(object): + + MAXAGE = 3600 * 24 * 365 # one year + + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self._cc_parsed = WeakKeyDictionary() + + def _parse_cachecontrol(self, r): + if r not in self._cc_parsed: + cch = r.headers.get('Cache-Control', '') + self._cc_parsed[r] = parse_cachecontrol(cch) + return self._cc_parsed[r] + + def should_cache_request(self, request): + if urlparse_cached(request).scheme in self.ignore_schemes: + return False + cc = self._parse_cachecontrol(request) + # obey user-agent directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Any other is eligible for caching + return True + + def should_cache_response(self, response, request): + # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # Status code 206 is not included because cache can not deal with partial contents + cc = self._parse_cachecontrol(response) + # obey directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Never cache 304 (Not Modified) responses + elif response.status == 304: + return False + # Any hint on response expiration is good + elif 'max-age' in cc or 'Expires' in response.headers: + return True + # Firefox fallbacks this statuses to one year expiration if none is set + elif response.status in (300, 301, 308): + return True + # Other statuses without expiration requires at least one validator + elif response.status in (200, 203, 401): + return 'Last-Modified' in response.headers or 'ETag' in response.headers + # Any other is probably not eligible for caching + # Makes no sense to cache responses that does not contain expiration + # info and can not be revalidated + else: + return False + + def is_cached_response_fresh(self, cachedresponse, request): + cc = self._parse_cachecontrol(cachedresponse) + ccreq = self._parse_cachecontrol(request) + if 'no-cache' in cc or 'no-cache' in ccreq: + return False + + now = time() + freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) + currentage = self._compute_current_age(cachedresponse, request, now) + if currentage < freshnesslifetime: + return True + # Cached response is stale, try to set validators if any + self._set_conditional_validators(request, cachedresponse) + return False + + def is_cached_response_valid(self, cachedresponse, response, request): + return response.status == 304 + + def _set_conditional_validators(self, request, cachedresponse): + if 'Last-Modified' in cachedresponse.headers: + request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + + if 'ETag' in cachedresponse.headers: + request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + + def _compute_freshness_lifetime(self, response, request, now): + # Reference nsHttpResponseHead::ComputeFresshnessLifetime + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 + cc = self._parse_cachecontrol(response) + if 'max-age' in cc: + try: + return max(0, int(cc['max-age'])) + except ValueError: + pass + + # Parse date header or synthesize it if none exists + date = rfc1123_to_epoch(response.headers.get('Date')) or now + + # Try HTTP/1.0 Expires header + if 'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers['Expires']) + # When parsing Expires header fails RFC 2616 section 14.21 says we + # should treat this as an expiration time in the past. + return max(0, expires - date) if expires else 0 + + # Fallback to heuristic using last-modified header + # This is not in RFC but on Firefox caching implementation + lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + if lastmodified and lastmodified <= date: + return (date - lastmodified) / 10 + + # This request can be cached indefinitely + if response.status in (300, 301, 308): + return self.MAXAGE + + # Insufficient information to compute fresshness lifetime + return 0 + + def _compute_current_age(self, response, request, now): + # Reference nsHttpResponseHead::ComputeCurrentAge + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html + currentage = 0 + # If Date header is not set we assume it is a fast connection, and + # clock is in sync with the server + date = rfc1123_to_epoch(response.headers.get('Date')) or now + if now > date: + currentage = now - date + + if 'Age' in response.headers: + try: + age = int(response.headers['Age']) + currentage = max(currentage, age) + except ValueError: + pass + + return currentage class DbmCacheStorage(object): @@ -51,11 +200,13 @@ class DbmCacheStorage(object): key = self._request_key(request) db = self.db tkey = '%s_time' % key - if not db.has_key(tkey): + if tkey not in db: return # not found + ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): return # expired + return pickle.loads(db['%s_data' % key]) def _request_key(self, request): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f7e04465f..c368f5ae7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,5 +1,5 @@ """ -This module contains the default values for all settings used by Scrapy. +This module contains the default values for all settings used by Scrapy. For more information about these settings you can read the settings documentation in docs/topics/settings.rst @@ -141,7 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' +HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.RFC2616Policy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 00baf4748..8944ded0a 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -16,7 +16,7 @@ from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware class _BaseTest(unittest.TestCase): storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' + policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) @@ -121,7 +121,9 @@ class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' -class DefaultMiddlewaretest(_BaseTest): +class DummyPolicyTest(_BaseTest): + + policy_class = 'scrapy.contrib.httpcache.DummyPolicy' def test_middleware(self): with self._middleware() as mw: @@ -211,14 +213,9 @@ class DefaultMiddlewaretest(_BaseTest): assert 'cached' in response.flags -class DummyMiddlewareTest(DefaultStorageTest): +class RFC2616PolicyTest(DefaultStorageTest): - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - - -class RFC2616MiddlewareTest(DefaultStorageTest): - - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' + policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' def _process_requestresponse(self, mw, request, response): try: