RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement single HTTP cache policy

This commit is contained in:
Pedro Faustino 2012-12-28 13:28:35 +01:00
parent 63d0b9f8c8
commit 3e31d06872
3 changed files with 45 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
scrapy/contrib/downloadermiddleware/httpcache.py
View File

@ -16,23 +16,44 @@ from scrapy.utils.misc import load_object
from scrapy.utils.project import data_path
class HttpCacheMiddleware(object):
class HttpCachePolicy(object):
def __init__(self, settings):
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
self.policy = settings.get('HTTPCACHE_POLICY')
if self.policy == 'dummy':
self.use_dummy_cache = True
else:
self.use_dummy_cache = False
def should_cache_response(self, response):
retval = response.status not in self.ignore_http_codes
if not self.use_dummy_cache and response.headers.has_key('cache-control'):
retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1)
#retval = retval and self.policy_response(response)
return retval
def should_cache_request(self, request):
retval = urlparse_cached(request).scheme not in self.ignore_schemes
if not self.use_dummy_cache and request.headers.has_key('cache-control'):
retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1)
#retval = retval and self.policy_request(request)
return retval
class HttpCacheMiddleware(HttpCachePolicy):
def __init__(self, settings, stats):
if not settings.getbool('HTTPCACHE_ENABLED'):
raise NotConfigured
self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
self.ignore_http_codes = map(int, settings.getlist('HTTPCACHE_IGNORE_HTTP_CODES'))
self.use_dummy_cache = settings.getbool('HTTPCACHE_USE_DUMMY')
self.stats = stats
self.policy_request = settings.get('HTTPCACHE_POLICY_REQUEST')
self.policy_response = settings.get('HTTPCACHE_POLICY_RESPONSE')
super(HttpCacheMiddleware, self).__init__(settings)
@classmethod
def from_crawler(cls, crawler):
o = cls(crawler.settings, crawler.stats)
o = cls.from_settings(crawler.settings, crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
return o
@ -44,7 +65,7 @@ class HttpCacheMiddleware(object):
self.storage.close_spider(spider)
def process_request(self, request, spider):
if not self.is_cacheable(request):
if not self.should_cache_request(request):
return
response = self.storage.retrieve_response(spider, request)
@ -54,7 +75,7 @@ class HttpCacheMiddleware(object):
self.stats.inc_value('httpcache/revalidation', spider=spider)
return
if response and self.is_cacheable_response(response):
if response and self.should_cache_response(response):
self.stats.inc_value('httpcache/hit', spider=spider)
if self.use_dummy_cache:
response.flags.append('cached')
@ -69,8 +90,8 @@ class HttpCacheMiddleware(object):
raise IgnoreRequest("Ignored request not in cache: %s" % request)
def process_response(self, request, response, spider):
if (self.is_cacheable(request)
and self.is_cacheable_response(response)):
if (self.should_cache_request(request)
and self.should_cache_response(response)):
if self.use_dummy_cache:
if 'cached' not in response.flags:
self.storage.store_response(spider, request, response)
@ -84,19 +105,6 @@ class HttpCacheMiddleware(object):
self.stats.inc_value('httpcache/hit', spider=spider)
return response
def is_cacheable_response(self, response):
retval = response.status not in self.ignore_http_codes
if not self.use_dummy_cache and response.headers.has_key('cache-control'):
retval = retval and (response.headers['cache-control'].lower().find('no-store') == -1)
retval = retval and self.policy_response(response)
return retval
def is_cacheable(self, request):
retval = urlparse_cached(request).scheme not in self.ignore_schemes
if not self.use_dummy_cache and request.headers.has_key('cache-control'):
retval = retval and (request.headers['cache-control'].lower().find('no-store') == -1)
retval = retval and self.policy_request(request)
return retval
class FilesystemCacheStorage(object):

4
scrapy/settings/default_settings.py
View File

@ -134,7 +134,6 @@ FEED_EXPORTERS_BASE = {
}
HTTPCACHE_ENABLED = False
HTTPCACHE_USE_DUMMY = True
HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_IGNORE_MISSING = False
HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage'
@ -142,8 +141,7 @@ HTTPCACHE_EXPIRATION_SECS = 0
HTTPCACHE_IGNORE_HTTP_CODES = []
HTTPCACHE_IGNORE_SCHEMES = ['file']
HTTPCACHE_DBM_MODULE = 'anydbm'
HTTPCACHE_POLICY_REQUEST = lambda request : True
HTTPCACHE_POLICY_RESPONSE = lambda response : True
HTTPCACHE_POLICY = 'dummy'
ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'

26
scrapy/tests/test_downloadermiddleware_httpcache.py
View File

@ -41,12 +41,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def _get_settings(self, **new_settings):
settings = {
'HTTPCACHE_ENABLED': True,
'HTTPCACHE_USE_DUMMY': True,
'HTTPCACHE_DIR': self.tmpdir,
'HTTPCACHE_EXPIRATION_SECS': 1,
'HTTPCACHE_IGNORE_HTTP_CODES': [],
'HTTPCACHE_POLICY_REQUEST': lambda request : True,
'HTTPCACHE_POLICY_RESPONSE': lambda response : True,
'HTTPCACHE_POLICY': 'dummy'
}
settings.update(new_settings)
return Settings(settings)
@ -54,7 +52,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
@contextmanager
def _storage(self, **new_settings):
settings = self._get_settings(**new_settings)
if settings.getbool('HTTPCACHE_USE_DUMMY'):
if settings.get('HTTPCACHE_POLICY') == 'dummy':
storage = self.storage_class(settings)
else:
storage = self.realcache_storage_class(settings)
@ -187,7 +185,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_middleware_response304_not_cached(self):
# test response is not cached because the status is 304 Not Modified
# (so it should be cached already)
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw:
assert mw.process_request(self.request, self.spider) is None
response = Response('http://www.example.com', status=304)
mw.process_response(self.request, response, self.spider)
@ -199,7 +197,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_middleware_response_nostore_not_cached(self):
# test response is not cached because of the Cache-Control 'no-store' directive
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw:
assert mw.process_request(self.request, self.spider) is None
response = Response('http://www.example.com', headers=
{'Content-Type': 'text/html', 'Cache-Control': 'no-store'},
@ -212,7 +210,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_middleware_request_nostore_not_cached(self):
# test response is not cached because of the request's Cache-Control 'no-store' directive
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw:
request = Request('http://www.example.com',
headers={'User-Agent': 'test', 'Cache-Control': 'no-store'})
assert mw.process_request(request, self.spider) is None
@ -223,14 +221,14 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_middleware_response_cached_and_fresh(self):
# test response cached and fresh
with self._middleware(HTTPCACHE_USE_DUMMY=False) as mw:
with self._middleware(HTTPCACHE_POLICY='rfc2616') as mw:
response = mw.process_response(self.request, self.response, self.spider)
self.assertRaises(IgnoreRequest, mw.process_request, self.request, self.spider)
assert 'cached' not in response.flags
def test_real_http_cache_middleware_response_cached_and_stale(self):
# test response cached but stale
with self._middleware(HTTPCACHE_USE_DUMMY=False,
with self._middleware(HTTPCACHE_POLICY='rfc2616',
HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmRealCacheStorage') as mw:
response = Response('http://www.example.com', headers=
{'Content-Type': 'text/html', 'Cache-Control': 'no-cache'},
@ -244,7 +242,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_storage_response_cached_and_fresh(self):
# test response is cached and is fresh
# (response requested should be same as response received)
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
with self._storage(HTTPCACHE_POLICY='rfc2616') as storage:
assert storage.retrieve_response(self.spider, self.request) is None
response = Response('http://www.example.com', headers=
@ -257,7 +255,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_storage_response403_cached_and_further_requests_ignored(self):
# test response is cached but further requests are ignored
# because response status is 403 (as per the RFC)
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
with self._storage(HTTPCACHE_POLICY='rfc2616') as storage:
assert storage.retrieve_response(self.spider, self.request) is None
response = Response('http://www.example.com', headers=
@ -270,7 +268,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_storage_response_cached_and_stale(self):
# test response is cached and is stale (no cache validators inserted)
# (request should be same as response received)
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
with self._storage(HTTPCACHE_POLICY='rfc2616') as storage:
assert storage.retrieve_response(self.spider, self.request) is None
response = Response('http://www.example.com', headers=
@ -283,7 +281,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_storage_response_cached_and_stale_with_cache_validators(self):
# test response is cached and is stale and cache validators are inserted
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
with self._storage(HTTPCACHE_POLICY='rfc2616') as storage:
assert storage.retrieve_response(self.spider, self.request) is None
response = Response('http://www.example.com', headers=
@ -297,7 +295,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
def test_real_http_cache_storage_response_cached_and_transparent(self):
# test response is not cached because of the request's Cache-Control 'no-cache' directive
# http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.2
with self._storage(HTTPCACHE_USE_DUMMY=False) as storage:
with self._storage(HTTPCACHE_POLICY='rfc2616') as storage:
request = Request('http://www.example.com',
headers={'User-Agent': 'test', 'Cache-Control': 'no-cache'})
assert storage.retrieve_response(self.spider, request) is None