From 0e5bea67fd41b970602cab5c97f69ce2728a8759 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 28 May 2009 00:27:42 -0300 Subject: [PATCH] images: adapt images pipeline to recent changes on HttpException topic --- scrapy/contrib/pipeline/images.py | 26 +++++++++++++------------- scrapy/contrib/pipeline/s3images.py | 11 ++++++----- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/scrapy/contrib/pipeline/images.py b/scrapy/contrib/pipeline/images.py index d3fcb3d76..0b97f947d 100644 --- a/scrapy/contrib/pipeline/images.py +++ b/scrapy/contrib/pipeline/images.py @@ -62,20 +62,25 @@ class BaseImagesPipeline(MediaPipeline): # ('270', (270, 270)) ) - - def media_downloaded(self, response, request, info): mtype = self.MEDIA_NAME referer = request.headers.get('Referer') - if not response or not response.body: - msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: no-content' \ + if response.status != 200: + msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \ + % (mtype, request, referer, errmsg) + log.msg(msg, level=log.WARNING, domain=info.domain) + raise ImageException(msg) + + if not response.body: + msg = 'Image (empty-content): Empty %s from %s referred in <%s>: no-content' \ % (mtype, request, referer) log.msg(msg, level=log.WARNING, domain=info.domain) raise ImageException(msg) - status = 'cached' if getattr(response, 'cached', False) else 'downloaded' - msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer) + status = 'cached' if 'cached' in response.flags else 'downloaded' + msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % \ + (status, mtype, request, referer) log.msg(msg, level=log.DEBUG, domain=info.domain) self.inc_stats(info.domain, status) @@ -93,13 +98,8 @@ class BaseImagesPipeline(MediaPipeline): def media_failed(self, failure, request, info): referer = request.headers.get('Referer') - if isinstance(failure.value, (HttpException, IgnoreRequest)): - errmsg = str(failure.value) - else: - errmsg = str(failure) - - msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \ - % (self.MEDIA_NAME, request, referer, errmsg) + msg = 'Image (unknow-error): Error downloading %s from %s referred in <%s>: %s' \ + % (self.MEDIA_NAME, request, referer, str(failure)) log.msg(msg, level=log.WARNING, domain=info.domain) raise ImageException(msg) diff --git a/scrapy/contrib/pipeline/s3images.py b/scrapy/contrib/pipeline/s3images.py index 3adbd8066..0bde3be67 100644 --- a/scrapy/contrib/pipeline/s3images.py +++ b/scrapy/contrib/pipeline/s3images.py @@ -59,11 +59,12 @@ class S3ImagesPipeline(BaseImagesPipeline): def stat_key(self, key, info): def _onsuccess(response): - checksum = response.headers['Etag'].strip('"') - last_modified = response.headers['Last-Modified'] - modified_tuple = rfc822.parsedate_tz(last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - return {'checksum': checksum, 'last_modified': modified_stamp} + if response.status == 200: + checksum = response.headers['Etag'].strip('"') + last_modified = response.headers['Last-Modified'] + modified_tuple = rfc822.parsedate_tz(last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + return {'checksum': checksum, 'last_modified': modified_stamp} req = self.s3_request(key, method='HEAD') dfd = self.s3_download(req, info)