move FilesystemCacheStorage to scrapy.contrib.httpcache

scrapy/contrib/downloadermiddleware/httpcache.py

@@ -1,18 +1,7 @@
-import os
-from os.path import join, exists
-from time import time
-import cPickle as pickle
-
-from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
-
 from scrapy import signals
-from scrapy.http import Headers
 from scrapy.exceptions import NotConfigured, IgnoreRequest
-from scrapy.responsetypes import responsetypes
-from scrapy.utils.request import request_fingerprint
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
-from scrapy.utils.project import data_path
 
 
 class HttpCacheMiddleware(object):
@@ -67,71 +56,16 @@ class HttpCacheMiddleware(object):
         return urlparse_cached(request).scheme not in self.ignore_schemes
 
 
-class FilesystemCacheStorage(object):
+from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage
 
-    def __init__(self, settings):
-        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
-        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
 
-    def open_spider(self, spider):
-        pass
+class FilesystemCacheStorage(_FilesystemCacheStorage):
 
-    def close_spider(self, spider):
-        pass
-
-    def retrieve_response(self, spider, request):
-        """Return response if present in cache, or None otherwise."""
-        metadata = self._read_meta(spider, request)
-        if metadata is None:
-            return # not cached
-        rpath = self._get_request_path(spider, request)
-        with open(join(rpath, 'response_body'), 'rb') as f:
-            body = f.read()
-        with open(join(rpath, 'response_headers'), 'rb') as f:
-            rawheaders = f.read()
-        url = metadata.get('response_url')
-        status = metadata['status']
-        headers = Headers(headers_raw_to_dict(rawheaders))
-        respcls = responsetypes.from_args(headers=headers, url=url)
-        response = respcls(url=url, headers=headers, status=status, body=body)
-        return response
-
-    def store_response(self, spider, request, response):
-        """Store the given response in the cache."""
-        rpath = self._get_request_path(spider, request)
-        if not exists(rpath):
-            os.makedirs(rpath)
-        metadata = {
-            'url': request.url,
-            'method': request.method,
-            'status': response.status,
-            'response_url': response.url,
-            'timestamp': time(),
-        }
-        with open(join(rpath, 'meta'), 'wb') as f:
-            f.write(repr(metadata))
-        with open(join(rpath, 'pickled_meta'), 'wb') as f:
-            pickle.dump(metadata, f, protocol=2)
-        with open(join(rpath, 'response_headers'), 'wb') as f:
-            f.write(headers_dict_to_raw(response.headers))
-        with open(join(rpath, 'response_body'), 'wb') as f:
-            f.write(response.body)
-        with open(join(rpath, 'request_headers'), 'wb') as f:
-            f.write(headers_dict_to_raw(request.headers))
-        with open(join(rpath, 'request_body'), 'wb') as f:
-            f.write(request.body)
-
-    def _get_request_path(self, spider, request):
-        key = request_fingerprint(request)
-        return join(self.cachedir, spider.name, key[0:2], key)
-
-    def _read_meta(self, spider, request):
-        rpath = self._get_request_path(spider, request)
-        metapath = join(rpath, 'pickled_meta')
-        if not exists(metapath):
-            return # not found
-        mtime = os.stat(rpath).st_mtime
-        if 0 < self.expiration_secs < time() - mtime:
-            return # expired
-        with open(metapath, 'rb') as f:
-            return pickle.load(f)
+    def __init__(self, *args, **kwargs):
+        import warnings
+        from scrapy.exceptions import ScrapyDeprecationWarning
+        warnings.warn('Importing FilesystemCacheStorage from '
+                      'scrapy.contrib.downloadermiddlware.httpcache is '
+                      'deprecated, use scrapy.contrib.httpcache instead.',
+                      category=ScrapyDeprecationWarning, stacklevel=1)
+        super(_FilesystemCacheStorage, self).__init__(*args, **kwargs)

scrapy/contrib/httpcache.py

@@ -1,7 +1,7 @@
 import os
 from time import time
 import cPickle as pickle
-
+from w3lib.http import headers_raw_to_dict, headers_dict_to_raw
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.request import request_fingerprint
@@ -26,7 +26,7 @@ class DbmCacheStorage(object):
     def retrieve_response(self, spider, request):
         data = self._read_data(spider, request)
         if data is None:
-            return # not cached
+            return  # not cached
         url = data['url']
         status = data['status']
         headers = Headers(data['headers'])
@@ -59,3 +59,73 @@ class DbmCacheStorage(object):
 
     def _request_key(self, request):
         return request_fingerprint(request)
+
+
+class FilesystemCacheStorage(object):
+
+    def __init__(self, settings):
+        self.cachedir = data_path(settings['HTTPCACHE_DIR'])
+        self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS')
+
+    def open_spider(self, spider):
+        pass
+
+    def close_spider(self, spider):
+        pass
+
+    def retrieve_response(self, spider, request):
+        """Return response if present in cache, or None otherwise."""
+        metadata = self._read_meta(spider, request)
+        if metadata is None:
+            return  # not cached
+        rpath = self._get_request_path(spider, request)
+        with open(os.path.join(rpath, 'response_body'), 'rb') as f:
+            body = f.read()
+        with open(os.path.join(rpath, 'response_headers'), 'rb') as f:
+            rawheaders = f.read()
+        url = metadata.get('response_url')
+        status = metadata['status']
+        headers = Headers(headers_raw_to_dict(rawheaders))
+        respcls = responsetypes.from_args(headers=headers, url=url)
+        response = respcls(url=url, headers=headers, status=status, body=body)
+        return response
+
+    def store_response(self, spider, request, response):
+        """Store the given response in the cache."""
+        rpath = self._get_request_path(spider, request)
+        if not os.path.exists(rpath):
+            os.makedirs(rpath)
+        metadata = {
+            'url': request.url,
+            'method': request.method,
+            'status': response.status,
+            'response_url': response.url,
+            'timestamp': time(),
+        }
+        with open(os.path.join(rpath, 'meta'), 'wb') as f:
+            f.write(repr(metadata))
+        with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f:
+            pickle.dump(metadata, f, protocol=2)
+        with open(os.path.join(rpath, 'response_headers'), 'wb') as f:
+            f.write(headers_dict_to_raw(response.headers))
+        with open(os.path.join(rpath, 'response_body'), 'wb') as f:
+            f.write(response.body)
+        with open(os.path.join(rpath, 'request_headers'), 'wb') as f:
+            f.write(headers_dict_to_raw(request.headers))
+        with open(os.path.join(rpath, 'request_body'), 'wb') as f:
+            f.write(request.body)
+
+    def _get_request_path(self, spider, request):
+        key = request_fingerprint(request)
+        return os.path.join(self.cachedir, spider.name, key[0:2], key)
+
+    def _read_meta(self, spider, request):
+        rpath = self._get_request_path(spider, request)
+        metapath = os.path.join(rpath, 'pickled_meta')
+        if not os.path.exists(metapath):
+            return  # not found
+        mtime = os.stat(rpath).st_mtime
+        if 0 < self.expiration_secs < time() - mtime:
+            return  # expired
+        with open(metapath, 'rb') as f:
+            return pickle.load(f)

scrapy/tests/test_downloadermiddleware_httpcache.py

@@ -9,22 +9,24 @@ from scrapy.spider import BaseSpider
 from scrapy.settings import Settings
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.test import get_crawler
-from scrapy.contrib.downloadermiddleware.httpcache import \
-    FilesystemCacheStorage, HttpCacheMiddleware
+from scrapy.contrib.httpcache import FilesystemCacheStorage, DbmCacheStorage
+from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware
 
 
 class HttpCacheMiddlewareTest(unittest.TestCase):
 
-    storage_class = FilesystemCacheStorage
+    storage_class = DbmCacheStorage
 
     def setUp(self):
         self.crawler = get_crawler()
         self.spider = BaseSpider('example.com')
         self.tmpdir = tempfile.mkdtemp()
         self.request = Request('http://www.example.com',
-            headers={'User-Agent': 'test'})
-        self.response = Response('http://www.example.com', headers=
-            {'Content-Type': 'text/html'}, body='test body', status=202)
+                               headers={'User-Agent': 'test'})
+        self.response = Response('http://www.example.com',
+                                 headers={'Content-Type': 'text/html'},
+                                 body='test body',
+                                 status=202)
         self.crawler.stats.open_spider(self.spider)
 
     def tearDown(self):
@@ -177,5 +179,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase):
         self.assertEqual(response1.headers, response2.headers)
         self.assertEqual(response1.body, response2.body)
 
+
+class FilesystemCacheStorageTest(HttpCacheMiddlewareTest):
+
+    storage = FilesystemCacheStorage
+
 if __name__ == '__main__':
     unittest.main()