diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 6600dfc5d..289676218 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -4,6 +4,8 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ +from threading import Thread + from scrapy.command import ScrapyCommand from scrapy.shell import Shell from scrapy import log @@ -35,12 +37,11 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None - shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True, \ - code=opts.code) - def err(f): - log.err(f, "Shell error") - self.exitcode = 1 - d = shell.start(url=url) - d.addErrback(err) - d.addBoth(lambda _: self.crawler.stop()) - self.crawler.start() + shell = Shell(self.crawler, update_vars=self.update_vars, code=opts.code) + self._start_crawler_thread() + shell.start(url=url) + + def _start_crawler_thread(self): + t = Thread(target=self.crawler.start) + t.daemon = True + t.start() diff --git a/scrapy/shell.py b/scrapy/shell.py index 3def28290..4458b553d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -7,6 +7,7 @@ See documentation in docs/topics/shell.rst import signal from twisted.internet import reactor, threads +from twisted.python import threadable from w3lib.url import any_to_uri from scrapy.item import BaseItem @@ -25,24 +26,18 @@ class Shell(object): relevant_classes = (BaseSpider, Request, Response, BaseItem, \ XPathSelector, Settings) - def __init__(self, crawler, update_vars=None, inthread=False, code=None): + def __init__(self, crawler, update_vars=None, code=None): self.crawler = crawler self.update_vars = update_vars or (lambda x: None) self.item_class = load_object(crawler.settings['DEFAULT_ITEM_CLASS']) self.spider = None - self.inthread = inthread + self.inthread = not threadable.isInIOThread() self.code = code self.vars = {} - def start(self, *a, **kw): + def start(self, url=None, request=None, response=None, spider=None): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) - if self.inthread: - return threads.deferToThread(self._start, *a, **kw) - else: - self._start(*a, **kw) - - def _start(self, url=None, request=None, response=None, spider=None): if url: self.fetch(url, spider) elif request: