From c4434590b0c037b794751f2cf5d58e01cd18a9e4 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 24 Nov 2008 10:18:55 +0000 Subject: [PATCH] images pipeline factorization --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40401 --- .../trunk/scrapy/contrib/pipeline/images.py | 92 ++++++++++--------- scrapy/trunk/scrapy/contrib/pipeline/media.py | 22 ++--- 2 files changed, 61 insertions(+), 53 deletions(-) diff --git a/scrapy/trunk/scrapy/contrib/pipeline/images.py b/scrapy/trunk/scrapy/contrib/pipeline/images.py index e9f06cc03..87de0a761 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/images.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/images.py @@ -20,22 +20,60 @@ from scrapy.contrib.pipeline.media import MediaPipeline IMAGE_EXPIRES = settings.getint('IMAGES_EXPIRES', 90) class NoimagesDrop(DropItem): - pass + """Product with no images exception""" class ImageException(Exception): """General image error exception""" -class ImagesPipeline(MediaPipeline): +class BaseImagesPipeline(MediaPipeline): MEDIA_TYPE = 'image' - THUMBS = None -# THUMBS = ( -# ("50", (50, 50)), -# ("110", (110, 110)), -# ("270", (270, 270)) -# ) + + def media_downloaded(self, response, request, info): + mtype = self.MEDIA_TYPE + referer = request.headers.get('Referer') + + if not response or not response.body.to_string(): + msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: no-content' \ + % (mtype, request, referer) + log.msg(msg, level=log.WARNING, domain=info.domain) + raise ImageException(msg) + + status = 'cached' if getattr(response, 'cached', False) else 'downloaded' + msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer) + log.msg(msg, level=log.DEBUG, domain=info.domain) + self.inc_stats(info.domain, status) + return self.image_downloaded(response, request, info) + + def media_failed(self, failure, request, info): + referer = request.headers.get('Referer') + errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure) + msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' \ + % (self.MEDIA_TYPE, request, referer, errmsg) + log.msg(msg, level=log.WARNING, domain=info.domain) + raise ImageException(msg) + + def inc_stats(self, domain, status): + stats.incpath('%s/image_count' % domain) + stats.incpath('%s/image_status_count/%s' % (domain, status)) + + def image_downloaded(self, response, request, info): + raise NotImplementedError + + +class ImagesPipeline(BaseImagesPipeline): MIN_WIDTH = 0 MIN_HEIGHT = 0 + THUMBS = ( +# ('50', (50, 50)), +# ('110', (110, 110)), +# ('270', (270, 270)) + ) + + class DomainInfo(BaseImagesPipeline.DomainInfo): + def __init__(self, domain): + super(ImagesPipeline.DomainInfo, self).__init__(domain) + self.created_directories = set() def __init__(self): if not settings['IMAGES_DIR']: @@ -46,7 +84,7 @@ class ImagesPipeline(MediaPipeline): self.MIN_WIDTH = settings.getint('IMAGES_MIN_WIDTH', 0) self.MIN_HEIGHT = settings.getint('IMAGES_MIN_HEIGHT', 0) - MediaPipeline.__init__(self) + super(ImagesPipeline, self).__init__() def media_to_download(self, request, info): relative, absolute = self._get_paths(request) @@ -57,31 +95,7 @@ class ImagesPipeline(MediaPipeline): (self.MEDIA_TYPE, request, referer), level=log.DEBUG, domain=info.domain) return relative - def media_downloaded(self, response, request, info): - mtype = self.MEDIA_TYPE - referer = request.headers.get('Referer') - - if not response or not response.body.to_string(): - msg = 'Image (empty): Empty %s (no content) in %s referred in <%s>: Empty image (no-content)' % (mtype, request, referer) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - result = self.save_image(response, request, info) # save and thumbs response - - status = 'cached' if getattr(response, 'cached', False) else 'downloaded' - msg = 'Image (%s): Downloaded %s from %s referred in <%s>' % (status, mtype, request, referer) - log.msg(msg, level=log.DEBUG, domain=info.domain) - self.inc_stats(info.domain, status) - return result - - def media_failed(self, failure, request, info): - referer = request.headers.get('Referer') - errmsg = str(failure.value) if isinstance(failure.value, HttpException) else str(failure) - msg = 'Image (http-error): Error downloading %s from %s referred in <%s>: %s' % (self.MEDIA_TYPE, request, referer, errmsg) - log.msg(msg, level=log.WARNING, domain=info.domain) - raise ImageException(msg) - - def save_image(self, response, request, info): + def image_downloaded(self, response, request, info): mtype = self.MEDIA_TYPE relpath, abspath = self._get_paths(request) dirname = os.path.dirname(abspath) @@ -94,7 +108,8 @@ class ImagesPipeline(MediaPipeline): raise ex except Exception, ex: referer = request.headers.get('Referer') - msg = 'Image (processing-error): Error thumbnailing %s from %s referred in <%s>: %s' % (mtype, request, referer, ex) + msg = 'Image (processing-error): Error thumbnailing %s from %s referred in <%s>: %s' \ + % (mtype, request, referer, ex) log.msg(msg, level=log.WARNING, domain=info.domain) raise ImageException(msg) @@ -106,17 +121,12 @@ class ImagesPipeline(MediaPipeline): return relative, absolute def mkdir(self, dirname, info=None): - already_created = info.extra.setdefault('created_directories', set()) if info else set() + already_created = info.created_directories if info else set() if dirname not in already_created: if not os.path.exists(dirname): os.makedirs(dirname) already_created.add(dirname) - def inc_stats(self, domain, status): - stats.incpath('%s/image_count' % domain) - stats.incpath('%s/image_status_count/%s' % (domain, status)) - - def should_download(path): """Should the image downloader download the image to the location specified diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py index 7bcb8db45..f4332b8a3 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/media.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py @@ -6,22 +6,21 @@ from scrapy.core.engine import scrapyengine from scrapy.spider import spiders -class DomainInfo(object): - def __init__(self, domain): - self.domain = domain - self.spider = spiders.fromdomain(domain) - self.downloading = {} - self.downloaded = {} - self.waiting = {} - self.extra = {} - - class MediaPipeline(object): + + class DomainInfo(object): + def __init__(self, domain): + self.domain = domain + self.spider = spiders.fromdomain(domain) + self.downloading = {} + self.downloaded = {} + self.waiting = {} + def __init__(self): self.domaininfo = {} def open_domain(self, domain): - self.domaininfo[domain] = DomainInfo(domain) + self.domaininfo[domain] = self.DomainInfo(domain) def close_domain(self, domain): del self.domaininfo[domain] @@ -98,7 +97,6 @@ class MediaPipeline(object): info.downloading[fp] = (request, dwld) # fill downloading state data dwld.addBoth(_downloaded) # append post-download hook dwld.addErrback(_bugtrap) # catch media_downloaded and media_failed unhandled errors - # WARNING: dont be tempted to return 'dwld' here. # declare request in downloading state (None is used as place holder) info.downloading[fp] = None