From 5e2db86cde2160c2aecf2945e8868a75af74932c Mon Sep 17 00:00:00 2001 From: samus_ Date: Tue, 22 Jul 2008 17:53:37 +0000 Subject: [PATCH] fixed pickle --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4092 --- .../scrapy/contrib/downloadermiddleware/cache.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py index b1f67ec12..a97f7dae8 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py @@ -111,9 +111,9 @@ class Cache(object): os.makedirs(linkpath) def is_cached(self, domain, key): - pickled_meta = os.path.join(self.requestpath(domain, key), 'pickled_meta') - if os.path.exists(pickled_meta): - with open(pickled_meta, 'rb') as f: + requestpath = self.requestpath(domain, key) + if os.path.exists(requestpath): + with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: metadata = pickle.load(f) if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=settings.getint('CACHE2_EXPIRATION_SECS')): return True @@ -133,7 +133,7 @@ class Cache(object): requestpath = self.requestpath(domain, key) metadata = responsebody = responseheaders = None - with open(os.path.join(requestpath, 'pickled_meta'), 'rb') as f: + with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f: metadata = pickle.load(f) with open(os.path.join(requestpath, 'response_body')) as f: responsebody = f.read() @@ -169,8 +169,8 @@ class Cache(object): with open(os.path.join(requestpath, 'meta_data'), 'w') as f: f.write(repr(metadata)) # pickled metadata (to recover without using eval) - with open(os.path.join(requestpath, 'pickled_meta'), 'wb') as f: - pickle.dump(metadata, f, -1) + with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f: + pickle.dump(metadata, f) # response with open(os.path.join(requestpath, 'response_headers'), 'w') as f: f.write(headers_dict_to_raw(response.headers))