added DBM storage backend for HTTP cache

docs/topics/downloader-middleware.rst

@@ -233,8 +233,18 @@ HttpCacheMiddleware
     downloads every time) and for trying your spider offline, when an Internet
     connection is not available.
 
-File system storage
-~~~~~~~~~~~~~~~~~~~
+    Scrapy ships with two storage backends for the HTTP cache middleware:
+
+    * :ref:`httpcache-fs-backend`
+    * :ref:`httpcache-dbm-backend`
+
+    You can change the storage backend with the :setting:`HTTPCACHE_STORAGE`
+    setting. Or you can also implement your own backend.
+
+.. _httpcache-fs-backend:
+
+File system backend (default)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 By default, the :class:`HttpCacheMiddleware` uses a file system storage  with the following structure:
 
@@ -257,8 +267,19 @@ inefficient in many file systems). An example directory could be::
 
    /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7
 
-The cache storage backend can be changed with the :setting:`HTTPCACHE_STORAGE`
-setting, but no other backend is provided with Scrapy yet.
+.. _httpcache-dbm-backend:
+
+DBM storage backend
+~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.13
+
+A DBM_ storage backend is also available for the HTTP cache middleware. To use
+it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE`
+to ``scrapy.contrib.httpcache.DbmCacheStorage``.
+
+By default, it uses the anydbm_ module, but you can change it with the
+:setting:`HTTPCACHE_DBM_MODULE` setting.
 
 Settings
 ~~~~~~~~
@@ -346,6 +367,18 @@ Default: ``'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage
 
 The class which implements the cache storage backend.
 
+.. setting:: HTTPCACHE_DBM_MODULE
+
+HTTPCACHE_DBM_MODULE
+^^^^^^^^^^^^^^^^^^^^
+
+.. versionadded:: 0.13
+
+Default: ``'anydbm'``
+
+The database module to use in the :ref:`DBM storage backend
+<httpcache-dbm-backend>`. This setting is specific to the DBM backend.
+
 
 HttpCompressionMiddleware
 -------------------------
@@ -491,3 +524,6 @@ UserAgentMiddleware
    In order for a spider to override the default user agent, its `user_agent`
    attribute must be set.
 
+
+.. _DBM: http://en.wikipedia.org/wiki/Dbm
+.. _anydbm: http://docs.python.org/library/anydbm.html

scrapy/contrib/httpcache.py

@@ -0,0 +1,64 @@
+from __future__ import with_statement
+
+import os
+from time import time
+import cPickle as pickle
+
+from scrapy.http import Headers
+from scrapy.core.downloader.responsetypes import responsetypes
+from scrapy.utils.request import request_fingerprint
+from scrapy.utils.project import data_path
+from scrapy import conf
+
+
+class DbmCacheStorage(object):
+
+    def __init__(self, settings=conf.settings):
+        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
+        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
+        self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE'])
+        self.dbs = {}
+
+    def open_spider(self, spider):
+        dbpath = os.path.join(self.cachedir, '%s.db' % spider.name)
+        self.dbs[spider] = self.dbmodule.open(dbpath, 'c')
+
+    def close_spider(self, spider):
+        self.dbs[spider].close()
+
+    def retrieve_response(self, spider, request):
+        data = self._read_data(spider, request)
+        if data is None:
+            return # not cached
+        url = data['url']
+        status = data['status']
+        headers = Headers(data['headers'])
+        body = data['body']
+        respcls = responsetypes.from_args(headers=headers, url=url)
+        response = respcls(url=url, headers=headers, status=status, body=body)
+        return response
+
+    def store_response(self, spider, request, response):
+        key = self._request_key(request)
+        data = {
+            'status': response.status,
+            'url': response.url,
+            'headers': dict(response.headers),
+            'body': response.body,
+        }
+        self.dbs[spider]['%s_data' % key] = pickle.dumps(data, protocol=2)
+        self.dbs[spider]['%s_time' % key] = str(time())
+
+    def _read_data(self, spider, request):
+        key = self._request_key(request)
+        db = self.dbs[spider]
+        tkey = '%s_time' % key
+        if not db.has_key(tkey):
+            return # not found
+        ts = db[tkey]
+        if 0 < self.expiration_secs < time() - float(ts):
+            return # expired
+        return pickle.loads(db['%s_data' % key])
+
+    def _request_key(self, request):
+        return request_fingerprint(request)

scrapy/settings/default_settings.py

@@ -155,6 +155,7 @@ HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCac
 HTTPCACHE_EXPIRATION_SECS = 0
 HTTPCACHE_IGNORE_HTTP_CODES = []
 HTTPCACHE_IGNORE_SCHEMES = ['file']
+HTTPCACHE_DBM_MODULE = 'anydbm'
 
 ITEM_PROCESSOR = 'scrapy.contrib.pipeline.ItemPipelineManager'