diff --git a/scrapy/trunk/scrapy/contrib/pipeline/images.py b/scrapy/trunk/scrapy/contrib/pipeline/images.py index e9f06cc03..5f83822fc 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/images.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/images.py @@ -4,6 +4,7 @@ import time import hashlib import urllib import urlparse +from datetime import datetime from cStringIO import StringIO import Image @@ -17,7 +18,7 @@ from scrapy.conf import settings from scrapy.contrib.pipeline.media import MediaPipeline # the age at which we download images again -IMAGE_EXPIRES = settings.getint('IMAGES_EXPIRES', 90) +IMAGE_EXPIRES = settings.getint('IMAGES_EXPIRES', 15) class NoimagesDrop(DropItem): pass @@ -50,12 +51,10 @@ class ImagesPipeline(MediaPipeline): def media_to_download(self, request, info): relative, absolute = self._get_paths(request) - if not should_download(absolute): - self.inc_stats(info.domain, 'uptodate') - referer = request.headers.get('Referer') - log.msg('Image (uptodate): Downloaded %s from %s referred in <%s>' % \ - (self.MEDIA_TYPE, request, referer), level=log.DEBUG, domain=info.domain) - return relative + expired, mtime = image_expired(absolute) + if not expired: + mod_date = datetime.utcfromtimestamp(mtime) + request.headers['If-Modified-Since'] = mod_date.strftime('%a, %d %b %Y %H:%M:%S GMT') def media_downloaded(self, response, request, info): mtype = self.MEDIA_TYPE @@ -75,8 +74,15 @@ class ImagesPipeline(MediaPipeline): return result def media_failed(self, failure, request, info): + mtype = self.MEDIA_TYPE referer = request.headers.get('Referer') errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure) + + if '304 Not Modified' in errmsg: + msg = 'Image (notmodified): Downloaded %s from %s referred in <%s>' % (mtype, request, referer) + log.msg(msg, level=log.DEBUG, domain=info.domain) + return self._get_paths(request)[0] + msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg) log.msg(msg, level=log.WARNING, domain=info.domain) raise ImageException(msg) @@ -118,16 +124,17 @@ class ImagesPipeline(MediaPipeline): -def should_download(path): - """Should the image downloader download the image to the location specified +def image_expired(path): + """ + Is the image older than the expiration time? """ try: mtime = os.path.getmtime(path) age_seconds = time.time() - mtime age_days = age_seconds / 60 / 60 / 24 - return age_days > IMAGE_EXPIRES + return (age_days > IMAGE_EXPIRES, mtime) except: - return True + return (True, 0) _MULTIPLE_SLASHES_REGEXP = re.compile(r"\/{2,}") _FINAL_SLASH_REGEXP = re.compile(r"\/$")