From ca695c8c79a0cb00f88486fcfc1e57b1ac33955e Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 15 Sep 2008 08:57:51 +0000 Subject: [PATCH] small bugfix and add bugtraps --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40235 --- scrapy/trunk/scrapy/contrib/pipeline/media.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/contrib/pipeline/media.py b/scrapy/trunk/scrapy/contrib/pipeline/media.py index 632f947d8..9c636a1dd 100644 --- a/scrapy/trunk/scrapy/contrib/pipeline/media.py +++ b/scrapy/trunk/scrapy/contrib/pipeline/media.py @@ -35,6 +35,9 @@ class MediaPipeline(object): assert urls is None or hasattr(urls, '__iter__'), \ 'get_urls_from_item should return None or iterable' + def _bugtrap(_failure, request): + log.msg('MediaPipeline Unhandled ERROR in %s: %s' % (request, _failure), log.ERROR, domain=domain) + lst = [] for url in urls or (): request = url if isinstance(url, Request) else Request(url=url) @@ -45,6 +48,7 @@ class MediaPipeline(object): errback=self.failed_item_media, errbackArgs=(item, request, info), ) + dfd.addErrback(_bugtrap, request) lst.append(dfd) dlst = defer.DeferredList(lst, consumeErrors=False) @@ -68,16 +72,21 @@ class MediaPipeline(object): return wad def _download(self, request, info, fp): - dwld = mustbe_deferred(self.download(request, info)) + def _bugtrap(_failure, request): + log.msg('MediaPipeline Unhandled ERROR in %s: %s' % (request, _failure), log.ERROR, domain=info.domain) + + dwld = mustbe_deferred(self.download, request, info) dwld.addCallbacks( callback=self.media_downloaded, callbackArgs=(request, info), errback=self.media_failure, errbackArgs=(request, info), ) + dwld.addErrback(_bugtrap, request) dwld.addBoth(self._downloaded, info, fp) info.downloading[fp] = (request, dwld) + def _downloaded(self, result, info, fp): info.downloaded[fp] = result # cache result waiting = info.waiting[fp] # client list