diff --git a/scrapy/contrib/downloadermiddleware/httpcache.py b/scrapy/contrib/downloadermiddleware/httpcache.py index f0df8fbbd..93a4129e5 100644 --- a/scrapy/contrib/downloadermiddleware/httpcache.py +++ b/scrapy/contrib/downloadermiddleware/httpcache.py @@ -1,18 +1,7 @@ -import os -from os.path import join, exists -from time import time -import cPickle as pickle - -from w3lib.http import headers_dict_to_raw, headers_raw_to_dict - from scrapy import signals -from scrapy.http import Headers from scrapy.exceptions import NotConfigured, IgnoreRequest -from scrapy.responsetypes import responsetypes -from scrapy.utils.request import request_fingerprint from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object -from scrapy.utils.project import data_path class HttpCacheMiddleware(object): @@ -67,71 +56,16 @@ class HttpCacheMiddleware(object): return urlparse_cached(request).scheme not in self.ignore_schemes -class FilesystemCacheStorage(object): +from scrapy.contrib.httpcache import FilesystemCacheStorage as _FilesystemCacheStorage - def __init__(self, settings): - self.cachedir = data_path(settings['HTTPCACHE_DIR']) - self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') - def open_spider(self, spider): - pass +class FilesystemCacheStorage(_FilesystemCacheStorage): - def close_spider(self, spider): - pass - - def retrieve_response(self, spider, request): - """Return response if present in cache, or None otherwise.""" - metadata = self._read_meta(spider, request) - if metadata is None: - return # not cached - rpath = self._get_request_path(spider, request) - with open(join(rpath, 'response_body'), 'rb') as f: - body = f.read() - with open(join(rpath, 'response_headers'), 'rb') as f: - rawheaders = f.read() - url = metadata.get('response_url') - status = metadata['status'] - headers = Headers(headers_raw_to_dict(rawheaders)) - respcls = responsetypes.from_args(headers=headers, url=url) - response = respcls(url=url, headers=headers, status=status, body=body) - return response - - def store_response(self, spider, request, response): - """Store the given response in the cache.""" - rpath = self._get_request_path(spider, request) - if not exists(rpath): - os.makedirs(rpath) - metadata = { - 'url': request.url, - 'method': request.method, - 'status': response.status, - 'response_url': response.url, - 'timestamp': time(), - } - with open(join(rpath, 'meta'), 'wb') as f: - f.write(repr(metadata)) - with open(join(rpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, protocol=2) - with open(join(rpath, 'response_headers'), 'wb') as f: - f.write(headers_dict_to_raw(response.headers)) - with open(join(rpath, 'response_body'), 'wb') as f: - f.write(response.body) - with open(join(rpath, 'request_headers'), 'wb') as f: - f.write(headers_dict_to_raw(request.headers)) - with open(join(rpath, 'request_body'), 'wb') as f: - f.write(request.body) - - def _get_request_path(self, spider, request): - key = request_fingerprint(request) - return join(self.cachedir, spider.name, key[0:2], key) - - def _read_meta(self, spider, request): - rpath = self._get_request_path(spider, request) - metapath = join(rpath, 'pickled_meta') - if not exists(metapath): - return # not found - mtime = os.stat(rpath).st_mtime - if 0 < self.expiration_secs < time() - mtime: - return # expired - with open(metapath, 'rb') as f: - return pickle.load(f) + def __init__(self, *args, **kwargs): + import warnings + from scrapy.exceptions import ScrapyDeprecationWarning + warnings.warn('Importing FilesystemCacheStorage from ' + 'scrapy.contrib.downloadermiddlware.httpcache is ' + 'deprecated, use scrapy.contrib.httpcache instead.', + category=ScrapyDeprecationWarning, stacklevel=1) + super(_FilesystemCacheStorage, self).__init__(*args, **kwargs) diff --git a/scrapy/contrib/httpcache.py b/scrapy/contrib/httpcache.py index 623fc2bed..147dd680a 100644 --- a/scrapy/contrib/httpcache.py +++ b/scrapy/contrib/httpcache.py @@ -1,7 +1,7 @@ import os from time import time import cPickle as pickle - +from w3lib.http import headers_raw_to_dict, headers_dict_to_raw from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.utils.request import request_fingerprint @@ -26,7 +26,7 @@ class DbmCacheStorage(object): def retrieve_response(self, spider, request): data = self._read_data(spider, request) if data is None: - return # not cached + return # not cached url = data['url'] status = data['status'] headers = Headers(data['headers']) @@ -59,3 +59,73 @@ class DbmCacheStorage(object): def _request_key(self, request): return request_fingerprint(request) + + +class FilesystemCacheStorage(object): + + def __init__(self, settings): + self.cachedir = data_path(settings['HTTPCACHE_DIR']) + self.expiration_secs = settings.getint('HTTPCACHE_EXPIRATION_SECS') + + def open_spider(self, spider): + pass + + def close_spider(self, spider): + pass + + def retrieve_response(self, spider, request): + """Return response if present in cache, or None otherwise.""" + metadata = self._read_meta(spider, request) + if metadata is None: + return # not cached + rpath = self._get_request_path(spider, request) + with open(os.path.join(rpath, 'response_body'), 'rb') as f: + body = f.read() + with open(os.path.join(rpath, 'response_headers'), 'rb') as f: + rawheaders = f.read() + url = metadata.get('response_url') + status = metadata['status'] + headers = Headers(headers_raw_to_dict(rawheaders)) + respcls = responsetypes.from_args(headers=headers, url=url) + response = respcls(url=url, headers=headers, status=status, body=body) + return response + + def store_response(self, spider, request, response): + """Store the given response in the cache.""" + rpath = self._get_request_path(spider, request) + if not os.path.exists(rpath): + os.makedirs(rpath) + metadata = { + 'url': request.url, + 'method': request.method, + 'status': response.status, + 'response_url': response.url, + 'timestamp': time(), + } + with open(os.path.join(rpath, 'meta'), 'wb') as f: + f.write(repr(metadata)) + with open(os.path.join(rpath, 'pickled_meta'), 'wb') as f: + pickle.dump(metadata, f, protocol=2) + with open(os.path.join(rpath, 'response_headers'), 'wb') as f: + f.write(headers_dict_to_raw(response.headers)) + with open(os.path.join(rpath, 'response_body'), 'wb') as f: + f.write(response.body) + with open(os.path.join(rpath, 'request_headers'), 'wb') as f: + f.write(headers_dict_to_raw(request.headers)) + with open(os.path.join(rpath, 'request_body'), 'wb') as f: + f.write(request.body) + + def _get_request_path(self, spider, request): + key = request_fingerprint(request) + return os.path.join(self.cachedir, spider.name, key[0:2], key) + + def _read_meta(self, spider, request): + rpath = self._get_request_path(spider, request) + metapath = os.path.join(rpath, 'pickled_meta') + if not os.path.exists(metapath): + return # not found + mtime = os.stat(rpath).st_mtime + if 0 < self.expiration_secs < time() - mtime: + return # expired + with open(metapath, 'rb') as f: + return pickle.load(f) diff --git a/scrapy/tests/test_downloadermiddleware_httpcache.py b/scrapy/tests/test_downloadermiddleware_httpcache.py index fdabaa4ab..10b30326e 100644 --- a/scrapy/tests/test_downloadermiddleware_httpcache.py +++ b/scrapy/tests/test_downloadermiddleware_httpcache.py @@ -9,22 +9,24 @@ from scrapy.spider import BaseSpider from scrapy.settings import Settings from scrapy.exceptions import IgnoreRequest from scrapy.utils.test import get_crawler -from scrapy.contrib.downloadermiddleware.httpcache import \ - FilesystemCacheStorage, HttpCacheMiddleware +from scrapy.contrib.httpcache import FilesystemCacheStorage, DbmCacheStorage +from scrapy.contrib.downloadermiddleware.httpcache import HttpCacheMiddleware class HttpCacheMiddlewareTest(unittest.TestCase): - storage_class = FilesystemCacheStorage + storage_class = DbmCacheStorage def setUp(self): self.crawler = get_crawler() self.spider = BaseSpider('example.com') self.tmpdir = tempfile.mkdtemp() self.request = Request('http://www.example.com', - headers={'User-Agent': 'test'}) - self.response = Response('http://www.example.com', headers= - {'Content-Type': 'text/html'}, body='test body', status=202) + headers={'User-Agent': 'test'}) + self.response = Response('http://www.example.com', + headers={'Content-Type': 'text/html'}, + body='test body', + status=202) self.crawler.stats.open_spider(self.spider) def tearDown(self): @@ -177,5 +179,10 @@ class HttpCacheMiddlewareTest(unittest.TestCase): self.assertEqual(response1.headers, response2.headers) self.assertEqual(response1.body, response2.body) + +class FilesystemCacheStorageTest(HttpCacheMiddlewareTest): + + storage = FilesystemCacheStorage + if __name__ == '__main__': unittest.main()