From 752787e63454eb8732899255d9f4311f71bb4ede Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Thu, 6 Mar 2014 02:13:15 -0200
Subject: [PATCH 1/2] Add a LevelDB cache backend

---
 scrapy/contrib/httpcache.py                   | 63 +++++++++++++++++++
 .../test_downloadermiddleware_httpcache.py    |  7 +++
 tox.ini                                       |  1 +
 3 files changed, 71 insertions(+)

diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py
index 2099bab9c..549227424 100644
--- a/scrapy/contrib/httpcache.py
+++ b/scrapy/contrib/httpcache.py
@@ -285,6 +285,69 @@ class FilesystemCacheStorage(object):
             return pickle.load(f)
 
 
+class LeveldbCacheStorage(object):
+
+    def __init__(self, settings):
+        import leveldb
+        self._leveldb = leveldb
+        self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True)
+        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
+        self.db = None
+
+    def open_spider(self, spider):
+        dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name)
+        self.db = self._leveldb.LevelDB(dbpath)
+
+    def close_spider(self, spider):
+        del self.db
+
+    def retrieve_response(self, spider, request):
+        data = self._read_data(spider, request)
+        if data is None:
+            return  # not cached
+        url = data['url']
+        status = data['status']
+        headers = Headers(data['headers'])
+        body = data['body']
+        respcls = responsetypes.from_args(headers=headers, url=url)
+        response = respcls(url=url, headers=headers, status=status, body=body)
+        return response
+
+    def store_response(self, spider, request, response):
+        key = self._request_key(request)
+        data = {
+            'status': response.status,
+            'url': response.url,
+            'headers': dict(response.headers),
+            'body': response.body,
+        }
+        batch = self._leveldb.WriteBatch()
+        batch.Put('%s_data' % key, pickle.dumps(data, protocol=2))
+        batch.Put('%s_time' % key, str(time()))
+        self.db.Write(batch)
+
+    def _read_data(self, spider, request):
+        key = self._request_key(request)
+        try:
+            ts = self.db.Get('%s_time' % key)
+        except KeyError:
+            return  # not found or invalid entry
+
+        if 0 < self.expiration_secs < time() - float(ts):
+            return  # expired
+
+        try:
+            data = self.db.Get('%s_data' % key)
+        except KeyError:
+            return  # invalid entry
+        else:
+            return pickle.loads(data)
+
+    def _request_key(self, request):
+        return request_fingerprint(request)
+
+
+
 def parse_cachecontrol(header):
     """Parse Cache-Control header
 
diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py
index da769afbd..efd9a9b7a 100644
--- a/scrapy/tests/test_downloadermiddleware_httpcache.py
+++ b/scrapy/tests/test_downloadermiddleware_httpcache.py
@@ -5,6 +5,7 @@ import shutil
 import unittest
 import email.utils
 from contextlib import contextmanager
+import pytest
 
 from scrapy.http import Response, HtmlResponse, Request
 from scrapy.spider import Spider
@@ -136,6 +137,12 @@ class FilesystemStorageTest(DefaultStorageTest):
     storage_class = 'scrapy.contrib.httpcache.FilesystemCacheStorage'
 
 
+class LeveldbStorageTest(DefaultStorageTest):
+
+    pytest.importorskip('leveldb')
+    storage_class = 'scrapy.contrib.httpcache.LeveldbCacheStorage'
+
+
 class DummyPolicyTest(_BaseTest):
 
     policy_class = 'scrapy.contrib.httpcache.DummyPolicy'
diff --git a/tox.ini b/tox.ini
index b39ab2556..b230d42d3 100644
--- a/tox.ini
+++ b/tox.ini
@@ -15,6 +15,7 @@ deps =
     boto
     Pillow
     django
+    leveldb
     -rtests-requirements.txt
 commands =
     py.test --twisted {posargs:scrapy}

From a90fd81c97e32694164b71a4df18e446bd2e3ee2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= <dangra@gmail.com>
Date: Mon, 23 Jun 2014 14:28:55 -0300
Subject: [PATCH 2/2] add leveldb cache backend docs

---
 docs/topics/downloader-middleware.rst | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index da5a85655..e201260f9 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -422,6 +422,27 @@ In order to use this storage backend, set:
 
 * :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.DbmCacheStorage``
 
+.. _httpcache-storage-leveldb:
+
+LevelDB storage backend
+~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 0.23
+
+A LevelDB_ storage backend is also available for the HTTP cache middleware.
+
+This backend is not recommended for development because only one process can
+access LevelDB databases at the same time, so you can't run a crawl and open
+the scrapy shell in parallel for the same spider.
+
+In order to use this storage backend:
+
+* set :setting:`HTTPCACHE_STORAGE` to ``scrapy.contrib.httpcache.LeveldbCacheStorage``
+* install `LevelDB python bindings`_ like ``pip install leveldb``
+
+.. _LevelDB: http://code.google.com/p/leveldb/
+.. _leveldb python bindings: http://pypi.python.org/pypi/leveldb
+
 
 HTTPCache middleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~