Do leveldb compactation for the httpcache on closing.

This commit is contained in:
Rolando Espinoza 2015-06-11 13:27:49 -04:00 committed by Daniel Graña
parent a2dbde98d5
commit b285cebcbf
1 changed files with 3 additions and 0 deletions

View File

@ -302,6 +302,9 @@ class LeveldbCacheStorage(object):
self.db = self._leveldb.LevelDB(dbpath)
def close_spider(self, spider):
# Do compactation each time to save space and also recreate files to
# avoid them being removed in storages with timestamp-based autoremoval.
self.db.CompactRange()
del self.db
def retrieve_response(self, spider, request):