From 754d0f53f9d6381e9eeb2b87fe23a0c0ef01e6f1 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 23 Sep 2010 12:33:24 -0300 Subject: [PATCH] Fixed unbounded spider error in shell, and enclosed fetch() method in a try/except block for logging errors more reliably --- scrapy/shell.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index a4f4594b0..573a42147 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -56,23 +56,26 @@ class Shell(object): if spider is None: spider = create_spider_for_request(self.crawler.spiders, request, \ BaseSpider('default'), log_multiple=True) + spider.set_crawler(self.crawler) self.crawler.engine.open_spider(spider) return self.crawler.engine.schedule(request, spider) def fetch(self, request_or_url, spider=None): - if isinstance(request_or_url, Request): - request = request_or_url - url = request.url - else: - url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - response = None + # we enclose all this code in a try/except block to see errors when + # they happen in a thread try: + if isinstance(request_or_url, Request): + request = request_or_url + url = request.url + else: + url = any_to_uri(request_or_url) + request = Request(url, dont_filter=True) + response = None response = threads.blockingCallFromThread(reactor, \ self._schedule, request, spider) + self.populate_vars(url, response, request, spider) except: - log.err(Failure(), "Error fetching response", spider=spider) - self.populate_vars(url, response, request, spider) + log.err(Failure(), "Error fetching: %s" % request_or_url, spider=spider) def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class()