From 864a7aef87172ea3ff182ea56caa95e6912652d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 8 Jan 2013 17:26:32 -0200 Subject: [PATCH] More httpcache updates * Change default cache policy to RFC2616 * Update HttpCacheMiddleware documentation * Move policies to scrapy.contrib.httpcache * remove a lint error for .has_key() usage in DBM storage backend --- docs/topics/downloader-middleware.rst | 97 ++++++----- .../contrib/downloadermiddleware/httpcache.py | 147 ----------------- scrapy/contrib/httpcache.py | 155 +++++++++++++++++- scrapy/settings/default_settings.py | 4 +- .../test_downloadermiddleware_httpcache.py | 15 +- 5 files changed, 221 insertions(+), 197 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index a47ffa83e..395a7a817 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -285,6 +285,7 @@ HttpAuthMiddleware .. _Basic access authentication: http://en.wikipedia.org/wiki/Basic_access_authentication + HttpCacheMiddleware ------------------- @@ -298,64 +299,84 @@ HttpCacheMiddleware Scrapy ships with two HTTP cache storage backends: - * :ref:`httpcache-dbm-backend` - * :ref:`httpcache-fs-backend` + * :ref:`httpcache-storage-dbm` + * :ref:`httpcache-storage-fs` You can change the HTTP cache storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own storage backend. - - + Scrapy ships with two HTTP cache policies: - * Dummy policy - - This policy has no awareness of any HTTP Cache-Control - directives. Every request and its corresponding response are cached. - When the same request is seen again, the response is returned - without transferring anything from the Internet. - - The Dummy policy is useful for testing spiders faster (without having to - wait for downloads every time) and for trying your spider offline, when - an Internet connection is not available. The goal is to be able to - "replay" a spider run *exactly as it ran before*. - - This is the default cache policy. - - * RFC2616 policy - - This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP - Cache-Control awareness, aimed at production and used in continuous - runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). - - In order to use this policy, set: - - * :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy`` + * :ref:`httpcache-policy-rfc2616` + * :ref:`httpcache-policy-dummy` You can change the HTTP cache policy with the :setting:`HTTPCACHE_POLICY` setting. Or you can also implement your own policy. -.. _httpcache-dbm-backend: + +.. _httpcache-policy-rfc2616: + +RFC2616 policy (default) +~~~~~~~~~~~~~~~~~~~~~~~~ + +This policy provides a RFC2616 compliant HTTP cache, i.e. with HTTP +Cache-Control awareness, aimed at production and used in continuous +runs to avoid downloading unmodified data (to save bandwidth and speed up crawls). + +In order to use this policy, set: + +* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.RFC2616Policy`` + +This is the default cache policy. + + +.. _httpcache-policy-dummy: + +Dummy policy +~~~~~~~~~~~~ + +This policy has no awareness of any HTTP Cache-Control directives. +Every request and its corresponding response are cached. When the same +request is seen again, the response is returned without transferring +anything from the Internet. + +The Dummy policy is useful for testing spiders faster (without having +to wait for downloads every time) and for trying your spider offline, +when an Internet connection is not available. The goal is to be able to +"replay" a spider run *exactly as it ran before*. + +In order to use this policy, set: + +* :setting:`HTTPCACHE_POLICY` to ``scrapy.contrib.httpcache.DummyPolicy`` + + +.. _httpcache-storage-dbm: DBM storage backend (default) ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. versionadded:: 0.13 -A DBM_ storage backend is available for the HTTP cache middleware. To use it -(note: it is the default storage backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.httpcache.DbmCacheStorage``. +A DBM_ storage backend is available for the HTTP cache middleware. By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. -.. _httpcache-fs-backend: +In order to use this storage backend, set: -File system backend -~~~~~~~~~~~~~~~~~~~ +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.DbmCacheStorage`` + + +.. _httpcache-storage-fs: + +Filesystem storage backend +~~~~~~~~~~~~~~~~~~~~~~~~~~ A file system storage backend is also available for the HTTP cache middleware. -To use it (instead of the default DBM_ storage backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage``. + +In order to use this storage backend, set: + +* :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.FilesystemCacheStorage`` Each request/response pair is stored in a different directory containing the following files: @@ -376,6 +397,7 @@ inefficient in many file systems). An example directory could be:: /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 + HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -481,10 +503,11 @@ HTTPCACHE_POLICY .. versionadded:: 0.18 -Default: ``'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy'`` +Default: ``'scrapy.contrib.httpcache.RFC2616Policy'`` The class which implements the cache policy. + HttpCompressionMiddleware ------------------------- diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b3ac193b4..ebfd14c1c 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -8,153 +8,6 @@ from scrapy.utils.misc import load_object from scrapy.contrib.httpcache import rfc1123_to_epoch, parse_cachecontrol -class DummyPolicy(object): - - def __init__(self, settings): - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) - - def should_cache_request(self, request): - return urlparse_cached(request).scheme not in self.ignore_schemes - - def should_cache_response(self, response, request): - return response.status not in self.ignore_http_codes - - def is_cached_response_fresh(self, response, request): - return True - - def is_cached_response_valid(self, cachedresponse, response, request): - return True - - -class RFC2616Policy(object): - - MAXAGE = 3600 * 24 * 365 # one year - - def __init__(self, settings): - self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') - self._cc_parsed = WeakKeyDictionary() - - def _parse_cachecontrol(self, r): - if r not in self._cc_parsed: - cch = r.headers.get('Cache-Control', '') - self._cc_parsed[r] = parse_cachecontrol(cch) - return self._cc_parsed[r] - - def should_cache_request(self, request): - if urlparse_cached(request).scheme in self.ignore_schemes: - return False - cc = self._parse_cachecontrol(request) - # obey user-agent directive "Cache-Control: no-store" - if 'no-store' in cc: - return False - # Any other is eligible for caching - return True - - def should_cache_response(self, response, request): - # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 - # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 - # Status code 206 is not included because cache can not deal with partial contents - cc = self._parse_cachecontrol(response) - # obey directive "Cache-Control: no-store" - if 'no-store' in cc: - return False - # Never cache 304 (Not Modified) responses - elif response.status == 304: - return False - # Any hint on response expiration is good - elif 'max-age' in cc or 'Expires' in response.headers: - return True - # Firefox fallbacks this statuses to one year expiration if none is set - elif response.status in (300, 301, 308): - return True - # Other statuses without expiration requires at least one validator - elif response.status in (200, 203, 401): - return 'Last-Modified' in response.headers or 'ETag' in response.headers - # Any other is probably not eligible for caching - # Makes no sense to cache responses that does not contain expiration - # info and can not be revalidated - else: - return False - - def is_cached_response_fresh(self, cachedresponse, request): - cc = self._parse_cachecontrol(cachedresponse) - ccreq = self._parse_cachecontrol(request) - if 'no-cache' in cc or 'no-cache' in ccreq: - return False - - now = time() - freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) - currentage = self._compute_current_age(cachedresponse, request, now) - if currentage < freshnesslifetime: - return True - # Cached response is stale, try to set validators if any - self._set_conditional_validators(request, cachedresponse) - return False - - def is_cached_response_valid(self, cachedresponse, response, request): - return response.status == 304 - - def _set_conditional_validators(self, request, cachedresponse): - if 'Last-Modified' in cachedresponse.headers: - request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] - - if 'ETag' in cachedresponse.headers: - request.headers['If-None-Match'] = cachedresponse.headers['ETag'] - - def _compute_freshness_lifetime(self, response, request, now): - # Reference nsHttpResponseHead::ComputeFresshnessLifetime - # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 - cc = self._parse_cachecontrol(response) - if 'max-age' in cc: - try: - return max(0, int(cc['max-age'])) - except ValueError: - pass - - # Parse date header or synthesize it if none exists - date = rfc1123_to_epoch(response.headers.get('Date')) or now - - # Try HTTP/1.0 Expires header - if 'Expires' in response.headers: - expires = rfc1123_to_epoch(response.headers['Expires']) - # When parsing Expires header fails RFC 2616 section 14.21 says we - # should treat this as an expiration time in the past. - return max(0, expires - date) if expires else 0 - - # Fallback to heuristic using last-modified header - # This is not in RFC but on Firefox caching implementation - lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) - if lastmodified and lastmodified <= date: - return (date - lastmodified) / 10 - - # This request can be cached indefinitely - if response.status in (300, 301, 308): - return self.MAXAGE - - # Insufficient information to compute fresshness lifetime - return 0 - - def _compute_current_age(self, response, request, now): - # Reference nsHttpResponseHead::ComputeCurrentAge - # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html - currentage = 0 - # If Date header is not set we assume it is a fast connection, and - # clock is in sync with the server - date = rfc1123_to_epoch(response.headers.get('Date')) or now - if now > date: - currentage = now - date - - if 'Age' in response.headers: - try: - age = int(response.headers['Age']) - currentage = max(currentage, age) - except ValueError: - pass - - return currentage - - class HttpCacheMiddleware(object): def __init__(self, settings, stats): diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index d984ee8ed..0891c6355 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,12 +1,161 @@ import os -from time import time import cPickle as pickle +from time import time +from weakref import WeakKeyDictionary from email.utils import mktime_tz, parsedate_tz from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.project import data_path +from scrapy.utils.httpobj import urlparse_cached + + +class DummyPolicy(object): + + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES')) + + def should_cache_request(self, request): + return urlparse_cached(request).scheme not in self.ignore_schemes + + def should_cache_response(self, response, request): + return response.status not in self.ignore_http_codes + + def is_cached_response_fresh(self, response, request): + return True + + def is_cached_response_valid(self, cachedresponse, response, request): + return True + + +class RFC2616Policy(object): + + MAXAGE = 3600 * 24 * 365 # one year + + def __init__(self, settings): + self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') + self._cc_parsed = WeakKeyDictionary() + + def _parse_cachecontrol(self, r): + if r not in self._cc_parsed: + cch = r.headers.get('Cache-Control', '') + self._cc_parsed[r] = parse_cachecontrol(cch) + return self._cc_parsed[r] + + def should_cache_request(self, request): + if urlparse_cached(request).scheme in self.ignore_schemes: + return False + cc = self._parse_cachecontrol(request) + # obey user-agent directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Any other is eligible for caching + return True + + def should_cache_response(self, response, request): + # What is cacheable - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec14.9.1 + # Response cacheability - http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4 + # Status code 206 is not included because cache can not deal with partial contents + cc = self._parse_cachecontrol(response) + # obey directive "Cache-Control: no-store" + if 'no-store' in cc: + return False + # Never cache 304 (Not Modified) responses + elif response.status == 304: + return False + # Any hint on response expiration is good + elif 'max-age' in cc or 'Expires' in response.headers: + return True + # Firefox fallbacks this statuses to one year expiration if none is set + elif response.status in (300, 301, 308): + return True + # Other statuses without expiration requires at least one validator + elif response.status in (200, 203, 401): + return 'Last-Modified' in response.headers or 'ETag' in response.headers + # Any other is probably not eligible for caching + # Makes no sense to cache responses that does not contain expiration + # info and can not be revalidated + else: + return False + + def is_cached_response_fresh(self, cachedresponse, request): + cc = self._parse_cachecontrol(cachedresponse) + ccreq = self._parse_cachecontrol(request) + if 'no-cache' in cc or 'no-cache' in ccreq: + return False + + now = time() + freshnesslifetime = self._compute_freshness_lifetime(cachedresponse, request, now) + currentage = self._compute_current_age(cachedresponse, request, now) + if currentage < freshnesslifetime: + return True + # Cached response is stale, try to set validators if any + self._set_conditional_validators(request, cachedresponse) + return False + + def is_cached_response_valid(self, cachedresponse, response, request): + return response.status == 304 + + def _set_conditional_validators(self, request, cachedresponse): + if 'Last-Modified' in cachedresponse.headers: + request.headers['If-Modified-Since'] = cachedresponse.headers['Last-Modified'] + + if 'ETag' in cachedresponse.headers: + request.headers['If-None-Match'] = cachedresponse.headers['ETag'] + + def _compute_freshness_lifetime(self, response, request, now): + # Reference nsHttpResponseHead::ComputeFresshnessLifetime + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html#l259 + cc = self._parse_cachecontrol(response) + if 'max-age' in cc: + try: + return max(0, int(cc['max-age'])) + except ValueError: + pass + + # Parse date header or synthesize it if none exists + date = rfc1123_to_epoch(response.headers.get('Date')) or now + + # Try HTTP/1.0 Expires header + if 'Expires' in response.headers: + expires = rfc1123_to_epoch(response.headers['Expires']) + # When parsing Expires header fails RFC 2616 section 14.21 says we + # should treat this as an expiration time in the past. + return max(0, expires - date) if expires else 0 + + # Fallback to heuristic using last-modified header + # This is not in RFC but on Firefox caching implementation + lastmodified = rfc1123_to_epoch(response.headers.get('Last-Modified')) + if lastmodified and lastmodified <= date: + return (date - lastmodified) / 10 + + # This request can be cached indefinitely + if response.status in (300, 301, 308): + return self.MAXAGE + + # Insufficient information to compute fresshness lifetime + return 0 + + def _compute_current_age(self, response, request, now): + # Reference nsHttpResponseHead::ComputeCurrentAge + # http://dxr.mozilla.org/mozilla-central/netwerk/protocol/http/nsHttpResponseHead.cpp.html + currentage = 0 + # If Date header is not set we assume it is a fast connection, and + # clock is in sync with the server + date = rfc1123_to_epoch(response.headers.get('Date')) or now + if now > date: + currentage = now - date + + if 'Age' in response.headers: + try: + age = int(response.headers['Age']) + currentage = max(currentage, age) + except ValueError: + pass + + return currentage class DbmCacheStorage(object): @@ -51,11 +200,13 @@ class DbmCacheStorage(object): key = self._request_key(request) db = self.db tkey = '%s_time' % key - if not db.has_key(tkey): + if tkey not in db: return # not found + ts = db[tkey] if 0 < self.expiration_secs < time() - float(ts): return # expired + return pickle.loads(db['%s_data' % key]) def _request_key(self, request): diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index f7e04465f..c368f5ae7 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -1,5 +1,5 @@ """ -This module contains the default values for all settings used by Scrapy. +This module contains the default values for all settings used by Scrapy. For more information about these settings you can read the settings documentation in docs/topics/settings.rst @@ -141,7 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] HTTPCACHE_DBM_MODULE = 'anydbm' -HTTPCACHE_POLICY = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' +HTTPCACHE_POLICY = 'scrapy.contrib.httpcache.RFC2616Policy' ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager' diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 00baf4748..8944ded0a 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -16,7 +16,7 @@ from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware class _BaseTest(unittest.TestCase): storage_class = 'scrapy.contrib.httpcache.DbmCacheStorage' - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' + policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' def setUp(self): self.yesterday = email.utils.formatdate(time.time() - 86400) @@ -121,7 +121,9 @@ class FilesystemStorageTest(DefaultStorageTest): storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage' -class DefaultMiddlewaretest(_BaseTest): +class DummyPolicyTest(_BaseTest): + + policy_class = 'scrapy.contrib.httpcache.DummyPolicy' def test_middleware(self): with self._middleware() as mw: @@ -211,14 +213,9 @@ class DefaultMiddlewaretest(_BaseTest): assert 'cached' in response.flags -class DummyMiddlewareTest(DefaultStorageTest): +class RFC2616PolicyTest(DefaultStorageTest): - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.DummyPolicy' - - -class RFC2616MiddlewareTest(DefaultStorageTest): - - policy_class = 'scrapy.contrib.downloadermiddleware.httpcache.RFC2616Policy' + policy_class = 'scrapy.contrib.httpcache.RFC2616Policy' def _process_requestresponse(self, mw, request, response): try: