mirror of https://github.com/scrapy/scrapy.git
Add cache storage support for real HTTP caching. Add real HTTP caching unit tests for both middleware and cache storage.
This commit is contained in:
Pedro Faustino
parent
0e435fb5f9
commit
b2d3f4dd1b
|
|
@ -1,8 +1,12 @@
|
|||
import os
|
||||
import calendar
|
||||
import email.utils
|
||||
from time import time
|
||||
import cPickle as pickle
|
||||
|
||||
from scrapy import log
|
||||
from scrapy.http import Headers
|
||||
from scrapy.exceptions import IgnoreRequest
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.utils.request import request_fingerprint
|
||||
from scrapy.utils.project import data_path
|
||||
|
|
@ -59,3 +63,141 @@ class DbmCacheStorage(object):
|
|||
|
||||
def _request_key(self, request):
|
||||
return request_fingerprint(request)
|
||||
|
||||
|
||||
class BaseRealCacheStorage(object):
|
||||
"""
|
||||
Most of the code was taken from the httplib2 (MIT License)
|
||||
https://code.google.com/p/httplib2/source/browse/python2/httplib2/__init__.py
|
||||
"""
|
||||
|
||||
def parse_cache_control(self, headers):
|
||||
retval = {}
|
||||
if headers.has_key('cache-control'):
|
||||
parts = headers['cache-control'].split(',')
|
||||
parts_with_args = [tuple([x.strip().lower() for x in part.split("=", 1)]) for part in parts if -1 != part.find("=")]
|
||||
parts_wo_args = [(name.strip().lower(), 1) for name in parts if -1 == name.find("=")]
|
||||
retval = dict(parts_with_args + parts_wo_args)
|
||||
return retval
|
||||
|
||||
|
||||
def get(self, response_headers, request_headers):
|
||||
"""Determine freshness from the Date, Expires and Cache-Control headers.
|
||||
|
||||
We don't handle the following:
|
||||
|
||||
1. Cache-Control: max-stale
|
||||
2. Age: headers are not used in the calculations.
|
||||
|
||||
Not that this algorithm is simpler than you might think
|
||||
because we are operating as a private (non-shared) cache.
|
||||
This lets us ignore 's-maxage'. We can also ignore
|
||||
'proxy-invalidate' since we aren't a proxy.
|
||||
We will never return a stale document as
|
||||
fresh as a design decision, and thus the non-implementation
|
||||
of 'max-stale'. This also lets us safely ignore 'must-revalidate'
|
||||
since we operate as if every server has sent 'must-revalidate'.
|
||||
Since we are private we get to ignore both 'public' and
|
||||
'private' parameters. We also ignore 'no-transform' since
|
||||
we don't do any transformations.
|
||||
The 'no-store' parameter is handled at a higher level.
|
||||
So the only Cache-Control parameters we look at are:
|
||||
|
||||
no-cache
|
||||
only-if-cached
|
||||
max-age
|
||||
min-fresh
|
||||
"""
|
||||
|
||||
retval = "STALE"
|
||||
cc = self.parse_cache_control(request_headers)
|
||||
cc_response = self.parse_cache_control(response_headers)
|
||||
|
||||
if request_headers.has_key('pragma') and request_headers['pragma'].lower().find('no-cache') != -1:
|
||||
retval = "TRANSPARENT"
|
||||
if 'cache-control' not in request_headers:
|
||||
request_headers['cache-control'] = 'no-cache'
|
||||
elif cc.has_key('no-cache'):
|
||||
retval = "TRANSPARENT"
|
||||
elif cc_response.has_key('no-cache'):
|
||||
retval = "STALE"
|
||||
elif cc.has_key('only-if-cached'):
|
||||
retval = "FRESH"
|
||||
elif response_headers.has_key('date'):
|
||||
date = calendar.timegm(email.utils.parsedate_tz(response_headers['date']))
|
||||
now = time()
|
||||
current_age = max(0, now - date)
|
||||
if cc_response.has_key('max-age'):
|
||||
try:
|
||||
freshness_lifetime = int(cc_response['max-age'])
|
||||
except ValueError:
|
||||
freshness_lifetime = 0
|
||||
elif response_headers.has_key('expires'):
|
||||
expires = email.utils.parsedate_tz(response_headers['expires'])
|
||||
if None == expires:
|
||||
freshness_lifetime = 0
|
||||
else:
|
||||
freshness_lifetime = max(0, calendar.timegm(expires) - date)
|
||||
else:
|
||||
freshness_lifetime = 0
|
||||
if cc.has_key('max-age'):
|
||||
try:
|
||||
freshness_lifetime = int(cc['max-age'])
|
||||
except ValueError:
|
||||
freshness_lifetime = 0
|
||||
if cc.has_key('min-fresh'):
|
||||
try:
|
||||
min_fresh = int(cc['min-fresh'])
|
||||
except ValueError:
|
||||
min_fresh = 0
|
||||
current_age += min_fresh
|
||||
if freshness_lifetime > current_age:
|
||||
retval = "FRESH"
|
||||
return retval
|
||||
|
||||
def retrieve_cache(self, spider, request, response_headers, response_status, response_url='', response_body=''):
|
||||
# Determine our course of action:
|
||||
# Is the cached entry fresh or stale?
|
||||
#
|
||||
# There seems to be three possible answers:
|
||||
# 1. [FRESH] Return the Response object
|
||||
# 2. [STALE] Update the Request object with cache validators if available
|
||||
# 3. [TRANSPARENT] Don't update the Request with cache validators (Cache-Control: no-cache)
|
||||
entry_disposition = self.get(Headers(response_headers), Headers(request.headers))
|
||||
|
||||
# Per the RFC, requests should not be repeated in these situations
|
||||
if response_status in [400, 401, 403, 410]:
|
||||
raise IgnoreRequest("Ignored request because cached response status is %d." % response_status)
|
||||
|
||||
if entry_disposition == "FRESH":
|
||||
log.msg("Cache is FRESH", level=log.DEBUG, spider=spider)
|
||||
|
||||
headers = Headers(response_headers)
|
||||
respcls = responsetypes.from_args(headers=headers, url=response_url)
|
||||
response = respcls(url=response_url, headers=headers, status=response_status, body=response_body)
|
||||
|
||||
return response
|
||||
|
||||
new_request = request.copy()
|
||||
if entry_disposition == "STALE":
|
||||
log.msg("Cache is STALE, updating Request object with cache validators", level=log.DEBUG, spider=spider)
|
||||
if response_headers.has_key('ETag') and not 'If-None-Match' in request.headers:
|
||||
new_request.headers['If-None-Match'] = response_headers['ETag']
|
||||
if response_headers.has_key('Last-Modified') and not 'Last-Modified' in request.headers:
|
||||
new_request.headers['If-Modified-Since'] = response_headers['Last-Modified']
|
||||
elif entry_disposition == "TRANSPARENT":
|
||||
log.msg("Cache is TRANSPARENT, not adding cache validators to Request object", level=log.DEBUG, spider=spider)
|
||||
|
||||
return new_request
|
||||
|
||||
|
||||
class DbmRealCacheStorage(DbmCacheStorage, BaseRealCacheStorage):
|
||||
def __init__(self, settings):
|
||||
super(DbmRealCacheStorage, self).__init__(settings)
|
||||
|
||||
def retrieve_response(self, spider, request):
|
||||
data = self._read_data(spider, request)
|
||||
if data is None:
|
||||
return # not cached
|
||||
else:
|
||||
return self.retrieve_cache(spider, request, data['headers'], data['status'], data['url'], data['body'])
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import time
|
|||
import tempfile
|
||||
import shutil
|
||||
import unittest
|
||||
import email.utils
|
||||
from contextlib import contextmanager
|
||||
|
||||
from scrapy.http import Response, HtmlResponse, Request
|
||||
|
|
@ -11,11 +12,17 @@ from scrapy.exceptions import IgnoreRequest
|
|||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.contrib.downloadermiddleware.httpcache import \
|
||||
FilesystemCacheStorage, HttpCacheMiddleware
|
||||
from scrapy.contrib.httpcache import DbmRealCacheStorage
|
||||
|
||||
|
||||
class HttpCacheMiddlewareTest(unittest.TestCase):
|
||||
|
||||
storage_class = FilesystemCacheStorage
|
||||
realcache_storage_class = DbmRealCacheStorage
|
||||
|
||||
yesterday = email.utils.formatdate(time.time() - 1 * 24 * 60 * 60)
|
||||
now = email.utils.formatdate()
|
||||
tomorrow = email.utils.formatdate(time.time() + 1 * 24 * 60 * 60)
|
||||
|
||||
def setUp(self):
|
||||
self.crawler = get_crawler()
|
||||
|
|
@ -34,6 +41,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
def _get_settings(self, **new_settings):
|
||||
settings = {
|
||||
'HTTPCACHE_ENABLED': True,
|
||||
'HTTPCACHE_USE_DUMMY': True,
|
||||
'HTTPCACHE_DIR': self.tmpdir,
|
||||
'HTTPCACHE_EXPIRATION_SECS': 1,
|
||||
'HTTPCACHE_IGNORE_HTTP_CODES': [],
|
||||
|
|
@ -44,7 +52,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
@contextmanager
|
||||
def _storage(self, **new_settings):
|
||||
settings = self._get_settings(**new_settings)
|
||||
storage = self.storage_class(settings)
|
||||
if settings.getbool('HTTPCACHE_USE_DUMMY'):
|
||||
storage = self.storage_class(settings)
|
||||
else:
|
||||
storage = self.realcache_storage_class(settings)
|
||||
storage.open_spider(self.spider)
|
||||
try:
|
||||
yield storage
|
||||
|
|
@ -171,11 +182,146 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
|
|||
self.assertEqualResponse(self.response, response)
|
||||
assert 'cached' in response.flags
|
||||
|
||||
def test_real_http_cache_middleware_response304_not_cached(self):
|
||||
# test response is not cached because the status is 304 Not Modified
|
||||
# (so it should be cached already)
|
||||
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
response = Response('http://www.example.com', status=304)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
|
||||
assert 'cached' in response.flags
|
||||
assert mw.storage.retrieve_response(self.spider, self.request) is None
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
|
||||
def test_real_http_cache_middleware_response_nostore_not_cached(self):
|
||||
# test response is not cached because of the Cache-Control 'no-store' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Cache-Control': 'no-store'},
|
||||
body='test body', status=200)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, self.request) is None
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
|
||||
def test_real_http_cache_middleware_request_nostore_not_cached(self):
|
||||
# test response is not cached because of the request's Cache-Control 'no-store' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
|
||||
request = Request('http://www.example.com',
|
||||
headers={'User-Agent': 'test', 'Cache-Control': 'no-store'})
|
||||
assert mw.process_request(request, self.spider) is None
|
||||
mw.process_response(request, self.response, self.spider)
|
||||
|
||||
assert mw.storage.retrieve_response(self.spider, request) is None
|
||||
assert mw.process_request(request, self.spider) is None
|
||||
|
||||
def test_real_http_cache_middleware_response_cached_and_fresh(self):
|
||||
# test response cached and fresh
|
||||
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
|
||||
response = mw.process_response(self.request, self.response, self.spider)
|
||||
self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)
|
||||
assert 'cached' not in response.flags
|
||||
|
||||
def test_real_http_cache_middleware_response_cached_and_stale(self):
|
||||
# test response cached but stale
|
||||
with self._middleware(HTTPCACHE_USE_DUMMY=False,
|
||||
HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw:
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Cache-Control': 'no-cache'},
|
||||
body='test body', status=200)
|
||||
mw.process_response(self.request, response, self.spider)
|
||||
assert mw.process_request(self.request, self.spider) is None
|
||||
|
||||
response = mw.storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response, Request)
|
||||
|
||||
def test_real_http_cache_storage_response_cached_and_fresh(self):
|
||||
# test response is cached and is fresh
|
||||
# (response requested should be same as response received)
|
||||
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow},
|
||||
body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
self.assertEqualResponse(response, response2)
|
||||
|
||||
def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self):
|
||||
# test response is cached but further requests are ignored
|
||||
# because response status is 403 (as per the RFC)
|
||||
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.yesterday, 'Expires': self.tomorrow},
|
||||
body='test body', status=403)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
self.assertRaises(IgnoreRequest, storage.retrieve_response,
|
||||
self.spider, self.request)
|
||||
|
||||
def test_real_http_cache_storage_response_cached_and_stale(self):
|
||||
# test response is cached and is stale (no cache validators inserted)
|
||||
# (request should be same as response received)
|
||||
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday},
|
||||
body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response2, Request)
|
||||
self.assertEqualRequest(self.request, response2)
|
||||
|
||||
def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self):
|
||||
# test response is cached and is stale and cache validators are inserted
|
||||
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
|
||||
assert storage.retrieve_response(self.spider, self.request) is None
|
||||
|
||||
response = Response('http://www.example.com', headers=
|
||||
{'Content-Type': 'text/html', 'Date': self.now, 'Expires': self.yesterday,
|
||||
'Last-Modified': self.yesterday}, body='test body', status=200)
|
||||
storage.store_response(self.spider, self.request, response)
|
||||
response2 = storage.retrieve_response(self.spider, self.request)
|
||||
assert isinstance(response2, Request)
|
||||
self.assertEqualRequestButWithCacheValidators(self.request, response2)
|
||||
|
||||
def test_real_http_cache_storage_response_cached_and_transparent(self):
|
||||
# test response is not cached because of the request's Cache-Control 'no-cache' directive
|
||||
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
|
||||
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
|
||||
request = Request('http://www.example.com',
|
||||
headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'})
|
||||
assert storage.retrieve_response(self.spider, request) is None
|
||||
storage.store_response(self.spider, request, self.response)
|
||||
response = storage.retrieve_response(self.spider, request)
|
||||
assert isinstance(response, Request)
|
||||
self.assertEqualRequest(request, response)
|
||||
|
||||
def assertEqualResponse(self, response1, response2):
|
||||
self.assertEqual(response1.url, response2.url)
|
||||
self.assertEqual(response1.status, response2.status)
|
||||
self.assertEqual(response1.headers, response2.headers)
|
||||
self.assertEqual(response1.body, response2.body)
|
||||
|
||||
def assertEqualRequest(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
self.assertEqual(request1.headers, request2.headers)
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
def assertEqualRequestButWithCacheValidators(self, request1, request2):
|
||||
self.assertEqual(request1.url, request2.url)
|
||||
assert not request1.headers.has_key('If-None-Match')
|
||||
assert not request1.headers.has_key('If-Modified-Since')
|
||||
assert (request2.headers.has_key('If-None-Match') or \
|
||||
request2.headers.has_key('If-Modified-Since'))
|
||||
self.assertEqual(request1.body, request2.body)
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue