From f2966eebc7ecafa9562bf6707992c006aa6ff40b Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 30 Dec 2011 14:04:16 -0400 Subject: [PATCH 1/2] httpcache: keep stats of cache hit/miss/store. --- scrapy/contrib/downloadermiddleware/httpcache.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index b7c3bbd2e..045a1b2d0 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -11,6 +11,7 @@ from scrapy.xlib.pydispatch import dispatcher from scrapy import signals from scrapy.http import Headers from scrapy.exceptions import NotConfigured, IgnoreRequest +from scrapy.stats import stats from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached @@ -43,13 +44,17 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') + stats.inc_value('httpcache/hit', spider=spider) return response - elif self.ignore_missing: + + stats.inc_value('httpcache/miss', spider=spider) + if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): if self.is_cacheable(request) and self.is_cacheable_response(response): self.storage.store_response(spider, request, response) + stats.inc_value('httpcache/store', spider=spider) return response def is_cacheable_response(self, response): From 503fdf39fe6417436a675de09cfac9f2c4114194 Mon Sep 17 00:00:00 2001 From: Rolando Espinoza La fuente Date: Fri, 30 Dec 2011 14:21:07 -0400 Subject: [PATCH 2/2] httpcache: don't store already cached response. --- scrapy/contrib/downloadermiddleware/httpcache.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index 045a1b2d0..a3bae921c 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -52,7 +52,9 @@ class HttpCacheMiddleware(object): raise IgnoreRequest("Ignored request not in cache: %s" % request) def process_response(self, request, response, spider): - if self.is_cacheable(request) and self.is_cacheable_response(response): + if (self.is_cacheable(request) + and self.is_cacheable_response(response) + and 'cached' not in response.flags): self.storage.store_response(spider, request, response) stats.inc_value('httpcache/store', spider=spider) return response