diff --git a/docs/news.rst b/docs/news.rst index d9c95848e..214ae44a5 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -35,6 +35,7 @@ Scrapy changes: - replaced memory usage acounting with (more portable) `resource`_ module, removed ``scrapy.utils.memory`` module - removed signal: ``scrapy.mail.mail_sent`` - removed ``TRACK_REFS`` setting, now :ref:`trackrefs ` is always enabled +- DBM is now the default storage backend for HTTP cache middleware Scrapyd changes: diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1dd00e641..543cc3570 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -304,16 +304,30 @@ HttpCacheMiddleware Scrapy ships with two storage backends for the HTTP cache middleware: - * :ref:`httpcache-fs-backend` * :ref:`httpcache-dbm-backend` + * :ref:`httpcache-fs-backend` You can change the storage backend with the :setting:`HTTPCACHE_STORAGE` setting. Or you can also implement your own backend. +.. _httpcache-dbm-backend: + +DBM storage backend (default) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 0.13 + +A DBM_ storage backend is also available for the HTTP cache middleware. To use +it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE` +to ``scrapy.contrib.httpcache.DbmCacheStorage``. + +By default, it uses the anydbm_ module, but you can change it with the +:setting:`HTTPCACHE_DBM_MODULE` setting. + .. _httpcache-fs-backend: -File system backend (default) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +File system backend +~~~~~~~~~~~~~~~~~~~ By default, the :class:`HttpCacheMiddleware` uses a file system storage with the following structure: @@ -336,20 +350,6 @@ inefficient in many file systems). An example directory could be:: /path/to/cache/dir/example.com/72/72811f648e718090f041317756c03adb0ada46c7 -.. _httpcache-dbm-backend: - -DBM storage backend -~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 0.13 - -A DBM_ storage backend is also available for the HTTP cache middleware. To use -it (instead of the default filesystem backend) set :setting:`HTTPCACHE_STORAGE` -to ``scrapy.contrib.httpcache.DbmCacheStorage``. - -By default, it uses the anydbm_ module, but you can change it with the -:setting:`HTTPCACHE_DBM_MODULE` setting. - HTTPCache middleware settings ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index cb12ac031..f0df8fbbd 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -45,10 +45,10 @@ class HttpCacheMiddleware(object): response = self.storage.retrieve_response(spider, request) if response and self.is_cacheable_response(response): response.flags.append('cached') - self.stats.inc_value('httpcache/hit', spider=spider) + self.stats.inc_value('httpcache/hits', spider=spider) return response - self.stats.inc_value('httpcache/miss', spider=spider) + self.stats.inc_value('httpcache/misses', spider=spider) if self.ignore_missing: raise IgnoreRequest("Ignored request not in cache: %s" % request) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 87f980db2..623fc2bed 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -11,17 +11,17 @@ from scrapy.utils.project import data_path class DbmCacheStorage(object): def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) + self.cachedir = data_path(settings['HTTPCACHE_DIR'], createdir=True) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') self.dbmodule = __import__(settings['HTTPCACHE_DBM_MODULE']) - self.dbs = {} + self.db = None def open_spider(self, spider): dbpath = os.path.join(self.cachedir, '%s.db' % spider.name) - self.dbs[spider] = self.dbmodule.open(dbpath, 'c') + self.db = self.dbmodule.open(dbpath, 'c') def close_spider(self, spider): - self.dbs[spider].close() + self.db.close() def retrieve_response(self, spider, request): data = self._read_data(spider, request) @@ -43,12 +43,12 @@ class DbmCacheStorage(object): 'headers': dict(response.headers), 'body': response.body, } - self.dbs[spider]['%s_data' % key] = pickle.dumps(data, protocol=2) - self.dbs[spider]['%s_time' % key] = str(time()) + self.db['%s_data' % key] = pickle.dumps(data, protocol=2) + self.db['%s_time' % key] = str(time()) def _read_data(self, spider, request): key = self._request_key(request) - db = self.dbs[spider] + db = self.db tkey = '%s_time' % key if not db.has_key(tkey): return # not found diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index dd4b587ae..4f2730b98 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -137,7 +137,7 @@ FEED_EXPORTERS_BASE = { HTTPCACHE_ENABLED = False HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False -HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' +HTTPCACHE_STORAGE = 'scrapy.contrib.httpcache.DbmCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 HTTPCACHE_IGNORE_HTTP_CODES = [] HTTPCACHE_IGNORE_SCHEMES = ['file'] diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index 1474ff9a9..b4c6ad16f 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -38,7 +38,9 @@ class HttpCacheMiddlewareTest(unittest.TestCase): return self.storage_class(self._get_settings(**new_settings)) def _get_middleware(self, **new_settings): - return HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) + mw = HttpCacheMiddleware(self._get_settings(**new_settings), self.crawler.stats) + mw.spider_opened(self.spider) + return mw def test_storage(self): storage = self._get_storage() @@ -59,7 +61,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert storage.retrieve_response(self.spider, self.request) def test_middleware(self): - mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) + mw = self._get_middleware() assert mw.process_request(self.request, self.spider) is None mw.process_response(self.request, self.response, self.spider) response = mw.process_request(self.request, self.spider) @@ -68,7 +70,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): assert 'cached' in response.flags def test_different_request_response_urls(self): - mw = HttpCacheMiddleware(self._get_settings(), self.crawler.stats) + mw = self._get_middleware() req = Request('http://host.com/path') res = Response('http://host2.net/test.html') assert mw.process_request(req, self.spider) is None diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index fde7a4d09..a64fcda4f 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -37,11 +37,15 @@ def project_data_dir(project='default'): os.makedirs(d) return d -def data_path(path): +def data_path(path, createdir=False): """If path is relative, return the given path inside the project data dir, otherwise return the path unmodified """ - return path if isabs(path) else join(project_data_dir(), path) + if not isabs(path): + path = join(project_data_dir(), path) + if createdir and not exists(path): + os.makedirs(path) + return path def get_project_settings(): if ENVVAR not in os.environ: