make DBM the new default storage backend for HTTP cache middleware, simplified DBM storage backend code to avoid dealing with many spiders at once (not needed), and update httpcache stats names (hit -> hits, miss -> misses)

docs/news.rst

@@ -35,6 +35,7 @@ Scrapy changes:
 - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module
 - removed signal: ``scrapy.mail.mail_sent``
 - removed ``TRACK_REFS`` setting, now :ref:`trackrefs <topics-leaks-trackrefs>` is always enabled
+- DBM is now the default storage backend for HTTP cache middleware
 
 Scrapyd changes:
 

docs/topics/downloader-middleware.rst

@@ -304,16 +304,30 @@ HttpCacheMiddleware
 
     Scrapy ships with two storage backends for the HTTP cache middleware:
 
-    * :ref:`httpcache-fs-backend`
     * :ref:`httpcache-dbm-backend`
+    * :ref:`httpcache-fs-backend`
 
     You can change the storage backend with the :setting:`HTTPCACHE_STORAGE`
     setting. Or you can also implement your own backend.
 
+.. _httpcache-dbm-backend:
+
+DBM storage backend (default)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.13
+
+A DBM_ storage backend is also available for the HTTP cache middleware. To use
+it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE`
+to ``scrapy.contrib.httpcache.DbmCacheStorage``.
+
+By default, it uses the anydbm_ module, but you can change it with the
+:setting:`HTTPCACHE_DBM_MODULE` setting.
+
 .. _httpcache-fs-backend:
 
-File system backend (default)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+File system backend
+~~~~~~~~~~~~~~~~~~~
 
 By default, the :class:`HttpCacheMiddleware` uses a file system storage  with the following structure:
 
@@ -336,20 +350,6 @@ inefficient in many file systems). An example directory could be::
 
    /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
 
-.. _httpcache-dbm-backend:
-
-DBM storage backend
-~~~~~~~~~~~~~~~~~~~
-
-.. versionadded:: 0.13
-
-A DBM_ storage backend is also available for the HTTP cache middleware. To use
-it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE`
-to ``scrapy.contrib.httpcache.DbmCacheStorage``.
-
-By default, it uses the anydbm_ module, but you can change it with the
-:setting:`HTTPCACHE_DBM_MODULE` setting.
-
 HTTPCache middleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -45,10 +45,10 @@ class HttpCacheMiddleware(object):
         response = self.storage.retrieve_response(spider, request)
         if response and self.is_cacheable_response(response):
             response.flags.append('cached')
-            self.stats.inc_value('httpcache/hit', spider=spider)
+            self.stats.inc_value('httpcache/hits', spider=spider)
             return response
 
-        self.stats.inc_value('httpcache/miss', spider=spider)
+        self.stats.inc_value('httpcache/misses', spider=spider)
         if self.ignore_missing:
             raise IgnoreRequest("Ignored request not in cache: %s" % request)
 

scrapy/contrib/httpcache.py

@@ -11,17 +11,17 @@ from scrapy.utils.project import data_path
 class DbmCacheStorage(object):
 
     def __init__(self, settings):
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
+        self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
         self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'])
-        self.dbs = {}
+        self.db = None
 
     def open_spider(self, spider):
         dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
-        self.dbs[spider] = self.dbmodule.open(dbpath, 'c')
+        self.db = self.dbmodule.open(dbpath, 'c')
 
     def close_spider(self, spider):
-        self.dbs[spider].close()
+        self.db.close()
 
     def retrieve_response(self, spider, request):
         data = self._read_data(spider, request)
@@ -43,12 +43,12 @@ class DbmCacheStorage(object):
             'headers': dict(response.headers),
             'body': response.body,
         }
-        self.dbs[spider]['%s_data' % key] = pickle.dumps(data, protocol=2)
-        self.dbs[spider]['%s_time' % key] = str(time())
+        self.db['%s_data' % key] = pickle.dumps(data, protocol=2)
+        self.db['%s_time' % key] = str(time())
 
     def _read_data(self, spider, request):
         key = self._request_key(request)
-        db = self.dbs[spider]
+        db = self.db
         tkey = '%s_time' % key
         if not db.has_key(tkey):
             return # not found

scrapy/settings/default_settings.py

@@ -137,7 +137,7 @@ FEED_EXPORTERS_BASE = {
 HTTPCACHE_ENABLED = False
 HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
-HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'
+HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -38,7 +38,9 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         return self.storage_class(self._get_settings(**new_settings))
 
     def _get_middleware(self, **new_settings):
-        return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats)
+        mw = HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats)
+        mw.spider_opened(self.spider)
+        return mw
 
     def test_storage(self):
         storage = self._get_storage()
@@ -59,7 +61,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         assert storage.retrieve_response(self.spider, self.request)
 
     def test_middleware(self):
-        mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
+        mw = self._get_middleware()
         assert mw.process_request(self.request, self.spider) is None
         mw.process_response(self.request, self.response, self.spider)
         response = mw.process_request(self.request, self.spider)
@@ -68,7 +70,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         assert 'cached' in response.flags
 
     def test_different_request_response_urls(self):
-        mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats)
+        mw = self._get_middleware()
         req = Request('http://host.com/path')
         res = Response('http://host2.net/test.html')
         assert mw.process_request(req, self.spider) is None

scrapy/utils/project.py

@@ -37,11 +37,15 @@ def project_data_dir(project='default'):
         os.makedirs(d)
     return d
 
-def data_path(path):
+def data_path(path, createdir=False):
     """If path is relative, return the given path inside the project data dir,
     otherwise return the path unmodified
     """
-    return path if isabs(path) else join(project_data_dir(), path)
+    if not isabs(path):
+        path = join(project_data_dir(), path)
+    if createdir and not exists(path):
+        os.makedirs(path)
+    return path
 
 def get_project_settings():
     if ENVVAR not in os.environ: