diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6f1ccce68..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,3 @@ -import logging - from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -logger = logging.getLogger(__name__) - - class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -29,8 +24,6 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats - logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) - @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 247cac64e..8025efe77 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import gzip +import logging from six.moves import cPickle as pickle from importlib import import_module from time import time @@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +logger = logging.getLogger(__name__) + + class DummyPolicy(object): def __init__(self, settings): @@ -216,6 +220,8 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None + logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') @@ -271,6 +277,8 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): pass @@ -344,6 +352,8 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None + logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath)