From b285cebcbf1396df193361773dbd7fd893308469 Mon Sep 17 00:00:00 2001
From: Rolando Espinoza <darkrho@gmail.com>
Date: Thu, 11 Jun 2015 13:27:49 -0400
Subject: [PATCH] Do leveldb compactation for the httpcache on closing.

---
 scrapy/extensions/httpcache.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3173656fe..f31cb3278 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -302,6 +302,9 @@ class LeveldbCacheStorage(object):
         self.db = self._leveldb.LevelDB(dbpath)
 
     def close_spider(self, spider):
+        # Do compactation each time to save space and also recreate files to
+        # avoid them being removed in storages with timestamp-based autoremoval.
+        self.db.CompactRange()
         del self.db
 
     def retrieve_response(self, spider, request):