From c041328b45dfb062217dbe65848304a41ef3a576 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 9 Sep 2010 15:26:46 -0300 Subject: [PATCH] Fixed bug with thread unsafety calls from Scrapy shell --- scrapy/shell.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scrapy/shell.py b/scrapy/shell.py index cb299948a..12947f480 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -51,6 +51,13 @@ class Shell(object): self.populate_vars(request.url, response, request, spider) start_python_console(self.vars) + def _schedule(self, request, spider): + if spider is None: + spider = create_spider_for_request(self.crawler.spiders, request, \ + BaseSpider('default'), log_multiple=True) + self.crawler.engine.open_spider(spider) + return self.crawler.engine.schedule(request, spider) + def fetch(self, request_or_url, spider=None): if isinstance(request_or_url, Request): request = request_or_url @@ -58,14 +65,10 @@ class Shell(object): else: url = any_to_uri(request_or_url) request = Request(url, dont_filter=True) - if spider is None: - spider = create_spider_for_request(self.crawler.spiders, request, \ - BaseSpider('default'), log_multiple=True) - self.crawler.engine.open_spider(spider) response = None try: response = threads.blockingCallFromThread(reactor, \ - self.crawler.engine.schedule, request, spider) + self._schedule, request, spider) except: log.err(Failure(), "Error fetching response", spider=spider) self.populate_vars(url, response, request, spider)