From eed6eb49dade4ce780a6c633b6ee75904f57effe Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 17 Sep 2012 10:11:07 -0300 Subject: [PATCH] make DBM the new default storage backend for HTTP cache middleware, simplified DBM storage backend code to avoid dealing with many spiders at once (not needed), and update httpcache stats names (hit -> hits, miss -> misses) --- docs/news.rst | 1 + docs/topics/downloader-middleware.rst | 34 +++++++++---------- .../contrib/downloadermiddleware/httpcache.py | 4 +-- scrapy/contrib/httpcache.py | 14 ++++---- scrapy/settings/default_settings.py | 2 +- .../test_downloadermiddleware_httpcache.py | 8 +++-- scrapy/utils/project.py | 8 +++-- 7 files changed, 39 insertions(+), 32 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index d9c95848e..214ae44a5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -35,6 +35,7 @@ Scrapy changes: - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled +- DBM is now the default storage backend for HTTP cache middleware Scrapyd changes: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1dd00e641..543cc3570 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -304,16 +304,30 @@ HttpCacheMiddleware Scrapy ships with two storage backends for the HTTP cache middleware: - * :ref:`httpcache-fs-backend` * :ref:`httpcache-dbm-backend` + * :ref:`httpcache-fs-backend` You can change the storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own backend. +.. _httpcache-dbm-backend: + +DBM storage backend (default) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 0.13 + +A DBM_ storage backend is also available for the HTTP cache middleware. To use +it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE` +to ``scrapy.contrib.httpcache.DbmCacheStorage``. + +By default, it uses the anydbm_ module, but you can change it with the +:setting:`HTTPCACHE_DBM_MODULE` setting. + .. _httpcache-fs-backend: -File system backend (default) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +File system backend +~~~~~~~~~~~~~~~~~~~ By default, the :class:`HttpCacheMiddleware` uses a file system storage with the following structure: @@ -336,20 +350,6 @@ inefficient in many file systems). An example directory could be:: /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 -.. _httpcache-dbm-backend: - -DBM storage backend -~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 0.13 - -A DBM_ storage backend is also available for the HTTP cache middleware. To use -it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.httpcache.DbmCacheStorage``. - -By default, it uses the anydbm_ module, but you can change it with the -:setting:`HTTPCACHE_DBM_MODULE` setting. - HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index cb12ac031..f0df8fbbd 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -45,10 +45,10 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') - self.stats.inc_value('httpcache/hit', spider=spider) + self.stats.inc_value('httpcache/hits', spider=spider) return response - self.stats.inc_value('httpcache/miss', spider=spider) + self.stats.inc_value('httpcache/misses', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 87f980db2..623fc2bed 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -11,17 +11,17 @@ from scrapy.utils.project import data_path class DbmCacheStorage(object): def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) + self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE']) - self.dbs = {} + self.db = None def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) - self.dbs[spider] = self.dbmodule.open(dbpath, 'c') + self.db = self.dbmodule.open(dbpath, 'c') def close_spider(self, spider): - self.dbs[spider].close() + self.db.close() def retrieve_response(self, spider, request): data = self._read_data(spider, request) @@ -43,12 +43,12 @@ class DbmCacheStorage(object): 'headers': dict(response.headers), 'body': response.body, } - self.dbs[spider]['%s_data' % key] = pickle.dumps(data, protocol=2) - self.dbs[spider]['%s_time' % key] = str(time()) + self.db['%s_data' % key] = pickle.dumps(data, protocol=2) + self.db['%s_time' % key] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) - db = self.dbs[spider] + db = self.db tkey = '%s_time' % key if not db.has_key(tkey): return # not found diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index dd4b587ae..4f2730b98 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -137,7 +137,7 @@ FEED_EXPORTERS_BASE = { HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' +HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 1474ff9a9..b4c6ad16f 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -38,7 +38,9 @@ class HttpCacheMiddlewareTest(unittest.TestCase): return self.storage_class(self._get_settings(**new_settings)) def _get_middleware(self, **new_settings): - return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) + mw = HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) + mw.spider_opened(self.spider) + return mw def test_storage(self): storage = self._get_storage() @@ -59,7 +61,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert storage.retrieve_response(self.spider, self.request) def test_middleware(self): - mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) + mw = self._get_middleware() assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) @@ -68,7 +70,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert 'cached' in response.flags def test_different_request_response_urls(self): - mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) + mw = self._get_middleware() req = Request('http://host.com/path') res = Response('http://host2.net/test.html') assert mw.process_request(req, self.spider) is None diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fde7a4d09..a64fcda4f 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -37,11 +37,15 @@ def project_data_dir(project='default'): os.makedirs(d) return d -def data_path(path): +def data_path(path, createdir=False): """If path is relative, return the given path inside the project data dir, otherwise return the path unmodified """ - return path if isabs(path) else join(project_data_dir(), path) + if not isabs(path): + path = join(project_data_dir(), path) + if createdir and not exists(path): + os.makedirs(path) + return path def get_project_settings(): if ENVVAR not in os.environ: