RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed pickle 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%4092
This commit is contained in:
samus_ 2008-07-22 17:53:37 +00:00
parent 7f62faea5f
commit 5e2db86cde
1 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
View File

@ -111,9 +111,9 @@ class Cache(object):
os.makedirs(linkpath)
def is_cached(self, domain, key):
pickled_meta = os.path.join(self.requestpath(domain, key), 'pickled_meta')
if os.path.exists(pickled_meta):
with open(pickled_meta, 'rb') as f:
requestpath = self.requestpath(domain, key)
if os.path.exists(requestpath):
with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
metadata = pickle.load(f)
if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=settings.getint('CACHE2_EXPIRATION_SECS')):
return True
@ -133,7 +133,7 @@ class Cache(object):
requestpath = self.requestpath(domain, key)
metadata = responsebody = responseheaders = None
with open(os.path.join(requestpath, 'pickled_meta'), 'rb') as f:
with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
metadata = pickle.load(f)
with open(os.path.join(requestpath, 'response_body')) as f:
responsebody = f.read()
@ -169,8 +169,8 @@ class Cache(object):
with open(os.path.join(requestpath, 'meta_data'), 'w') as f:
f.write(repr(metadata))
# pickled metadata (to recover without using eval)
with open(os.path.join(requestpath, 'pickled_meta'), 'wb') as f:
pickle.dump(metadata, f, -1)
with open(os.path.join(requestpath, 'pickled_meta'), 'w') as f:
pickle.dump(metadata, f)
# response
with open(os.path.join(requestpath, 'response_headers'), 'w') as f:
f.write(headers_dict_to_raw(response.headers))