From 97d84d920bf97f3ec1becd291596b3f0ee966328 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 11:04:16 +0100 Subject: [PATCH 1/4] Logging the cache directory at HttpCacheMiddleware instantiation #2604 --- scrapy/downloadermiddlewares/httpcache.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 30e49b886..6f1ccce68 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,3 +1,5 @@ +import logging + from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -9,6 +11,9 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object +logger = logging.getLogger(__name__) + + class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -24,6 +29,8 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats + logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) + @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) From f96490df2ce532e3b8757b2464f5f3cc54181ce5 Mon Sep 17 00:00:00 2001 From: jorenham Date: Thu, 2 Mar 2017 16:17:51 +0100 Subject: [PATCH 2/4] Move cache storage logging to the individual storage classes --- scrapy/downloadermiddlewares/httpcache.py | 7 ------- scrapy/extensions/httpcache.py | 10 ++++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py index 6f1ccce68..30e49b886 100644 --- a/scrapy/downloadermiddlewares/httpcache.py +++ b/scrapy/downloadermiddlewares/httpcache.py @@ -1,5 +1,3 @@ -import logging - from email.utils import formatdate from twisted.internet import defer from twisted.internet.error import TimeoutError, DNSLookupError, \ @@ -11,9 +9,6 @@ from scrapy.exceptions import NotConfigured, IgnoreRequest from scrapy.utils.misc import load_object -logger = logging.getLogger(__name__) - - class HttpCacheMiddleware(object): DOWNLOAD_EXCEPTIONS = (defer.TimeoutError, TimeoutError, DNSLookupError, @@ -29,8 +24,6 @@ class HttpCacheMiddleware(object): self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.stats = stats - logger.debug("Using cache directory %(cachedir)s" % {'cachedir': self.storage.cachedir}) - @classmethod def from_crawler(cls, crawler): o = cls(crawler.settings, crawler.stats) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 247cac64e..8025efe77 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -1,6 +1,7 @@ from __future__ import print_function import os import gzip +import logging from six.moves import cPickle as pickle from importlib import import_module from time import time @@ -15,6 +16,9 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.python import to_bytes, to_unicode +logger = logging.getLogger(__name__) + + class DummyPolicy(object): def __init__(self, settings): @@ -216,6 +220,8 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None + logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') @@ -271,6 +277,8 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): pass @@ -344,6 +352,8 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None + logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) + def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) From 42b429dc37e17a3a24d9f5f6d9365b6d78479630 Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:15:59 +0100 Subject: [PATCH 3/4] Log full cache file path instead of cache directory for the storages that cache to single files. --- scrapy/extensions/httpcache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index 8025efe77..fe8c55c6b 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -220,12 +220,12 @@ class DbmCacheStorage(object): self.dbmodule = import_module(settings['HTTPCACHE_DBM_MODULE']) self.db = None - logger.debug("Using DBM cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) self.db = self.dbmodule.open(dbpath, 'c') + logger.debug("Using DBM cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): self.db.close() @@ -352,12 +352,12 @@ class LeveldbCacheStorage(object): self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.db = None - logger.debug("Using LevelDB cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.leveldb' % spider.name) self.db = self._leveldb.LevelDB(dbpath) + logger.debug("Using LevelDB cache storage in %(cachepath)s" % {'cachepath': dbpath}, extra={'spider': spider}) + def close_spider(self, spider): # Do compactation each time to save space and also recreate files to # avoid them being removed in storages with timestamp-based autoremoval. From 5e89db548419c4114ff71bd0504155c9a83868fa Mon Sep 17 00:00:00 2001 From: jorenham Date: Fri, 3 Mar 2017 15:32:20 +0100 Subject: [PATCH 4/4] Moved cache dir logging to `open_spider` in FilesystemCacheStorage for consistency --- scrapy/extensions/httpcache.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py index fe8c55c6b..2fb4b6a15 100644 --- a/scrapy/extensions/httpcache.py +++ b/scrapy/extensions/httpcache.py @@ -277,10 +277,9 @@ class FilesystemCacheStorage(object): self.use_gzip = settings.getbool('HTTPCACHE_GZIP') self._open = gzip.open if self.use_gzip else open - logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}) - def open_spider(self, spider): - pass + logger.debug("Using filesystem cache storage in %(cachedir)s" % {'cachedir': self.cachedir}, + extra={'spider': spider}) def close_spider(self, spider): pass