Some changes to HTTP Cache middleware:

* made it use the project data storage by default (closes #279)
* added HTTPCACHE_ENABLED setting (False by default) to enable it
* made HTTPCACHE_DIR = 'httpcache' by default (inside the project data storage)
* simplified HTTPCACHE_EXPIRATION_SECS semantics: zero means don't expire,
  dropped support for negative numbers
* other minor doc improvements

docs/topics/downloader-middleware.rst

@@ -230,8 +230,8 @@ HttpCacheMiddleware
     anything from the Internet.
 
     The HTTP cache is useful for testing spiders faster (without having to wait for
-    downloads every time) and for trying your spider offline, when you don't have
-    an Internet connection.
+    downloads every time) and for trying your spider offline, when an Internet
+    connection is not available.
 
 File system storage
 ~~~~~~~~~~~~~~~~~~~
@@ -266,15 +266,19 @@ Settings
 The :class:`HttpCacheMiddleware` can be configured through the following
 settings:
 
-.. setting:: HTTPCACHE_DIR
+.. setting:: HTTPCACHE_ENABLED
 
-HTTPCACHE_DIR
-^^^^^^^^^^^^^
+HTTPCACHE_ENABLED
+^^^^^^^^^^^^^^^^^
 
-Default: ``''`` (empty string)
+.. versionadded:: 0.11
 
-The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP
-cache will be disabled.
+Default: ``False``
+
+Whether the HTTP cache will be enabled.
+
+.. versionchanged:: 0.11
+   Before 0.11, :setting:`HTTPCACHE_DIR` was used to enable cache.
 
 .. setting:: HTTPCACHE_EXPIRATION_SECS
 
@@ -283,9 +287,24 @@ HTTPCACHE_EXPIRATION_SECS
 
 Default: ``0``
 
-Number of seconds to use for HTTP cache expiration. Requests that were cached
-before this time will be re-downloaded. If zero, cached requests will always
-expire. A negative number means requests will never expire.
+Expiration time for cached requests, in seconds.
+
+Cached requests older than this time will be re-downloaded. If zero, cached
+requests will never expire.
+
+.. versionchanged:: 0.11
+   Before 0.11, zero meant cached requests always expire.
+
+.. setting:: HTTPCACHE_DIR
+
+HTTPCACHE_DIR
+^^^^^^^^^^^^^
+
+Default: ``'httpcache'``
+
+The directory to use for storing the (low-level) HTTP cache. If empty, the HTTP
+cache will be disabled. If a relative path is given, is taken relative to the
+project data dir. For more info see: :ref:`topics-project-structure`.
 
 .. setting:: HTTPCACHE_IGNORE_HTTP_CODES
 

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -14,12 +14,15 @@ from scrapy.utils.request import request_fingerprint
 from scrapy.utils.http import headers_dict_to_raw, headers_raw_to_dict
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
+from scrapy.utils.project import data_path
 from scrapy import conf
 
 
 class HttpCacheMiddleware(object):
 
     def __init__(self, settings=conf.settings):
+        if not settings.getbool('HTTPCACHE_ENABLED'):
+            raise NotConfigured
         self.storage = load_object(settings['HTTPCACHE_STORAGE'])(settings)
         self.ignore_missing = settings.getbool('HTTPCACHE_IGNORE_MISSING')
         self.ignore_schemes = settings.getlist('HTTPCACHE_IGNORE_SCHEMES')
@@ -58,10 +61,7 @@ class HttpCacheMiddleware(object):
 class FilesystemCacheStorage(object):
 
     def __init__(self, settings=conf.settings):
-        cachedir = settings['HTTPCACHE_DIR']
-        if not cachedir:
-            raise NotConfigured
-        self.cachedir = cachedir
+        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
         self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
 
     def open_spider(self, spider):
@@ -123,7 +123,7 @@ class FilesystemCacheStorage(object):
         if not exists(metapath):
             return # not found
         mtime = os.stat(rpath).st_mtime
-        if 0 <= self.expiration_secs < time() - mtime:
+        if 0 < self.expiration_secs < time() - mtime:
             return # expired
         with open(metapath, 'rb') as f:
             return pickle.load(f)

scrapy/settings/default_settings.py

@@ -148,7 +148,8 @@ FEED_EXPORTERS_BASE = {
     'xml': 'scrapy.contrib.exporter.XmlItemExporter',
 }
 
-HTTPCACHE_DIR = ''
+HTTPCACHE_ENABLED = False
+HTTPCACHE_DIR = 'httpcache'
 HTTPCACHE_IGNORE_MISSING = False
 HTTPCACHE_STORAGE = 'scrapy.contrib.downloadermiddleware.httpcache.FilesystemCacheStorage'
 HTTPCACHE_EXPIRATION_SECS = 0

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -22,6 +22,7 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
 
     def _get_settings(self, **new_settings):
         settings = {
+            'HTTPCACHE_ENABLED': True,
             'HTTPCACHE_DIR': self.tmpdir,
             'HTTPCACHE_EXPIRATION_SECS': 1,
             'HTTPCACHE_IGNORE_HTTP_CODES': [],
@@ -46,17 +47,11 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         time.sleep(2) # wait for cache to expire
         assert storage.retrieve_response(self.spider, request2) is None
 
-    def test_storage_expire_immediately(self):
+    def test_storage_never_expire(self):
         storage = self._get_storage(HTTPCACHE_EXPIRATION_SECS=0)
         assert storage.retrieve_response(self.spider, self.request) is None
         storage.store_response(self.spider, self.request, self.response)
-        time.sleep(0.1) # required for win32
-        assert storage.retrieve_response(self.spider, self.request) is None
-
-    def test_storage_never_expire(self):
-        storage = self._get_storage(HTTPCACHE_EXPIRATION_SECS=-1)
-        assert storage.retrieve_response(self.spider, self.request) is None
-        storage.store_response(self.spider, self.request, self.response)
+        time.sleep(0.5) # give the chance to expire
         assert storage.retrieve_response(self.spider, self.request)
 
     def test_middleware(self):

scrapy/utils/project.py

@@ -22,13 +22,11 @@ def project_data_dir(project='default'):
         makedirs(d)
     return d
 
-def expand_data_path(path):
+def data_path(path):
     """If path is relative, return the given path inside the project data dir,
     otherwise return the path unmodified
     """
-    if isabs(path):
-        return path
-    return join(project_data_dir(), path)
+    return path if isabs(path) else join(project_data_dir(), path)
 
 def sqlite_db(path, nonwritable_fallback=True):
     """Get the SQLite database to use. If path is relative, returns the given
@@ -41,7 +39,7 @@ def sqlite_db(path, nonwritable_fallback=True):
     if not inside_project() or path == ':memory:':
         db = ':memory:'
     else:
-        db = expand_data_path(path)
+        db = data_path(path)
         if not is_writable(db) and nonwritable_fallback:
             warnings.warn("%r is not writable - using in-memory SQLite instead" % db)
             db = ':memory:'