From b488e3b4a90721debfff25e51011ba4886b81a7f Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Mon, 22 Jun 2009 14:24:04 -0300 Subject: [PATCH] catch downloader process_queue exceptions --- scrapy/core/downloader/manager.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index f99150612..d0d28642f 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -93,6 +93,12 @@ class Downloader(object): return deferred def process_queue(self, spider): + try: + self._process_queue(spider) + except: + log.exc('Downloader process queue bug') + + def _process_queue(self, spider): """ Effective download requests from site queue """ domain = spider.domain_name