RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

images: adapt images pipeline to recent changes on HttpException topic

This commit is contained in:
Daniel Grana 2009-05-28 00:27:42 -03:00
parent 7eaa3ed24d
commit 0e5bea67fd
2 changed files with 19 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
scrapy/contrib/pipeline/images.py
View File

@ -62,20 +62,25 @@ class BaseImagesPipeline(MediaPipeline):
# ('270', (270, 270))
)
def media_downloaded(self, response, request, info):
mtype = self.MEDIA_NAME
referer = request.headers.get('Referer')
if not response or not response.body:
msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: no-content' \
if response.status != 200:
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \
% (mtype, request, referer, errmsg)
log.msg(msg, level=log.WARNING, domain=info.domain)
raise ImageException(msg)
if not response.body:
msg = 'Image (empty-content): Empty %s from %s referred in <%s>: no-content' \
% (mtype, request, referer)
log.msg(msg, level=log.WARNING, domain=info.domain)
raise ImageException(msg)
status = 'cached' if getattr(response, 'cached', False) else 'downloaded'
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer)
status = 'cached' if 'cached' in response.flags else 'downloaded'
msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % \
(status, mtype, request, referer)
log.msg(msg, level=log.DEBUG, domain=info.domain)
self.inc_stats(info.domain, status)
@ -93,13 +98,8 @@ class BaseImagesPipeline(MediaPipeline):
def media_failed(self, failure, request, info):
referer = request.headers.get('Referer')
if isinstance(failure.value, (HttpException, IgnoreRequest)):
errmsg = str(failure.value)
else:
errmsg = str(failure)
msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \
% (self.MEDIA_NAME, request, referer, errmsg)
msg = 'Image (unknow-error): Error downloading %s from %s referred in <%s>: %s' \
% (self.MEDIA_NAME, request, referer, str(failure))
log.msg(msg, level=log.WARNING, domain=info.domain)
raise ImageException(msg)

11
scrapy/contrib/pipeline/s3images.py
View File

@ -59,11 +59,12 @@ class S3ImagesPipeline(BaseImagesPipeline):
def stat_key(self, key, info):
def _onsuccess(response):
checksum = response.headers['Etag'].strip('"')
last_modified = response.headers['Last-Modified']
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum, 'last_modified': modified_stamp}
if response.status == 200:
checksum = response.headers['Etag'].strip('"')
last_modified = response.headers['Last-Modified']
modified_tuple = rfc822.parsedate_tz(last_modified)
modified_stamp = int(rfc822.mktime_tz(modified_tuple))
return {'checksum': checksum, 'last_modified': modified_stamp}
req = self.s3_request(key, method='HEAD')
dfd = self.s3_download(req, info)