RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

small improvements to the cache 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40102
This commit is contained in:
samus_ 2008-07-24 01:52:28 +00:00
parent 8b8e3e7532
commit e86ee64727
2 changed files with 16 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/conf/core_settings.py
View File

@ -11,7 +11,7 @@ USER_AGENT = '%s/%s' % (BOT_NAME, BOT_VERSION)
DOWNLOAD_TIMEOUT = 180 # 3mins
CONCURRENT_DOMAINS = 8 # number of domains to scrape in parallel
REQUESTS_PER_DOMAIN = 8 # max simultaneous requests per domain
CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid
CACHE2_EXPIRATION_SECS = 48 * 60 * 60 # seconds while cached response is still valid (a negative value means "never expires")
LOG_ENABLED = True #
LOGLEVEL = 'DEBUG' # default loglevel

20
scrapy/trunk/scrapy/contrib/downloadermiddleware/cache.py
View File

@ -1,6 +1,7 @@
from __future__ import with_statement
import os
import sys
import hashlib
import datetime
import urlparse
@ -33,7 +34,11 @@ class CacheMiddleware(object):
key = request.fingerprint()
domain = spider.domain_name
response = self.cache.retrieve_response(domain, key)
try:
response = self.cache.retrieve_response(domain, key)
except:
log.msg("Corrupt cache for %s" % request.url, log.WARNING)
response = False
if response:
response.cached = True
if not 200 <= int(response.status) < 300:
@ -115,11 +120,16 @@ class Cache(object):
if os.path.exists(requestpath):
with open(os.path.join(requestpath, 'pickled_meta'), 'r') as f:
metadata = pickle.load(f)
if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=settings.getint('CACHE2_EXPIRATION_SECS')):
return True
expiration_secs = settings.getint('CACHE2_EXPIRATION_SECS')
if expiration_secs >= 0:
if datetime.datetime.utcnow() <= metadata['timestamp'] + datetime.timedelta(seconds=expiration_secs):
return True
else:
log.msg('dropping old cached response from %s' % metadata['timestamp'])
return False
else:
log.msg('dropping old cached response from %s' % metadata['timestamp'])
return False
# disabled cache expiration
return True
else:
return False