From 0f69e7a1915ccd2cc3c0107b452414e278edd79d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 1 Nov 2010 02:38:15 -0200 Subject: [PATCH] Some changes to HTTP Cache middleware: * made it use the project data storage by default (closes #279) * added HTTPCACHE_ENABLED setting (False by default) to enable it * made HTTPCACHE_DIR = 'httpcache' by default (inside the project data storage) * simplified HTTPCACHE_EXPIRATION_SECS semantics: zero means don't expire, dropped support for negative numbers * other minor doc improvements --- docs/topics/downloader-middleware.rst | 41 ++++++++++++++----- .../contrib/downloadermiddleware/httpcache.py | 10 ++--- scrapy/settings/default_settings.py | 3 +- .../test_downloadermiddleware_httpcache.py | 11 ++--- scrapy/utils/project.py | 8 ++-- 5 files changed, 43 insertions(+), 30 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index eb7a33009..0887a5c40 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -230,8 +230,8 @@ HttpCacheMiddleware anything from the Internet. The HTTP cache is useful for testing spiders faster (without having to wait for - downloads every time) and for trying your spider offline, when you don't have - an Internet connection. + downloads every time) and for trying your spider offline, when an Internet + connection is not available. File system storage ~~~~~~~~~~~~~~~~~~~ @@ -266,15 +266,19 @@ Settings The :class:`HttpCacheMiddleware` can be configured through the following settings: -.. setting:: HTTPCACHE_DIR +.. setting:: HTTPCACHE_ENABLED -HTTPCACHE_DIR -^^^^^^^^^^^^^ +HTTPCACHE_ENABLED +^^^^^^^^^^^^^^^^^ -Default: ``''`` (empty string) +.. versionadded:: 0.11 -The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP -cache will be disabled. +Default: ``False`` + +Whether the HTTP cache will be enabled. + +.. versionchanged:: 0.11 + Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache. .. setting:: HTTPCACHE_EXPIRATION_SECS @@ -283,9 +287,24 @@ HTTPCACHE_EXPIRATION_SECS Default: ``0`` -Number of seconds to use for HTTP cache expiration. Requests that were cached -before this time will be re-downloaded. If zero, cached requests will always -expire. A negative number means requests will never expire. +Expiration time for cached requests, in seconds. + +Cached requests older than this time will be re-downloaded. If zero, cached +requests will never expire. + +.. versionchanged:: 0.11 + Before 0.11, zero meant cached requests always expire. + +.. setting:: HTTPCACHE_DIR + +HTTPCACHE_DIR +^^^^^^^^^^^^^ + +Default: ``'httpcache'`` + +The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP +cache will be disabled. If a relative path is given, is taken relative to the +project data dir. For more info see: :ref:`topics-project-structure`. .. setting:: HTTPCACHE_IGNORE_HTTP_CODES diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index f7a83e657..e27db5d80 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -14,12 +14,15 @@ from scrapy.utils.request import request_fingerprint from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object +from scrapy.utils.project import data_path from scrapy import conf class HttpCacheMiddleware(object): def __init__(self, settings=conf.settings): + if not settings.getbool('HTTPCACHE_ENABLED'): + raise NotConfigured self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings) self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING') self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES') @@ -58,10 +61,7 @@ class HttpCacheMiddleware(object): class FilesystemCacheStorage(object): def __init__(self, settings=conf.settings): - cachedir = settings['HTTPCACHE_DIR'] - if not cachedir: - raise NotConfigured - self.cachedir = cachedir + self.cachedir = data_path(settings['HTTPCACHE_DIR']) self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') def open_spider(self, spider): @@ -123,7 +123,7 @@ class FilesystemCacheStorage(object): if not exists(metapath): return # not found mtime = os.stat(rpath).st_mtime - if 0 <= self.expiration_secs < time() - mtime: + if 0 < self.expiration_secs < time() - mtime: return # expired with open(metapath, 'rb') as f: return pickle.load(f) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index aaf445c7f..015b1d911 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -148,7 +148,8 @@ FEED_EXPORTERS_BASE = { 'xml': 'scrapy.contrib.exporter.XmlItemExporter', } -HTTPCACHE_DIR = '' +HTTPCACHE_ENABLED = False +HTTPCACHE_DIR = 'httpcache' HTTPCACHE_IGNORE_MISSING = False HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage' HTTPCACHE_EXPIRATION_SECS = 0 diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index ae6339208..af54383f6 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -22,6 +22,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase): def _get_settings(self, **new_settings): settings = { + 'HTTPCACHE_ENABLED': True, 'HTTPCACHE_DIR': self.tmpdir, 'HTTPCACHE_EXPIRATION_SECS': 1, 'HTTPCACHE_IGNORE_HTTP_CODES': [], @@ -46,17 +47,11 @@ class HttpCacheMiddlewareTest(unittest.TestCase): time.sleep(2) # wait for cache to expire assert storage.retrieve_response(self.spider, request2) is None - def test_storage_expire_immediately(self): + def test_storage_never_expire(self): storage = self._get_storage(HTTPCACHE_EXPIRATION_SECS=0) assert storage.retrieve_response(self.spider, self.request) is None storage.store_response(self.spider, self.request, self.response) - time.sleep(0.1) # required for win32 - assert storage.retrieve_response(self.spider, self.request) is None - - def test_storage_never_expire(self): - storage = self._get_storage(HTTPCACHE_EXPIRATION_SECS=-1) - assert storage.retrieve_response(self.spider, self.request) is None - storage.store_response(self.spider, self.request, self.response) + time.sleep(0.5) # give the chance to expire assert storage.retrieve_response(self.spider, self.request) def test_middleware(self): diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py index 596d8380f..53ffad794 100644 --- a/scrapy/utils/project.py +++ b/scrapy/utils/project.py @@ -22,13 +22,11 @@ def project_data_dir(project='default'): makedirs(d) return d -def expand_data_path(path): +def data_path(path): """If path is relative, return the given path inside the project data dir, otherwise return the path unmodified """ - if isabs(path): - return path - return join(project_data_dir(), path) + return path if isabs(path) else join(project_data_dir(), path) def sqlite_db(path, nonwritable_fallback=True): """Get the SQLite database to use. If path is relative, returns the given @@ -41,7 +39,7 @@ def sqlite_db(path, nonwritable_fallback=True): if not inside_project() or path == ':memory:': db = ':memory:' else: - db = expand_data_path(path) + db = data_path(path) if not is_writable(db) and nonwritable_fallback: warnings.warn("%r is not writable - using in-memory SQLite instead" % db) db = ':memory:'