diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 20a82c6d9..ae4426d2f 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst from scrapy.command import ScrapyCommand from scrapy.shell import Shell +from scrapy import log class Command(ScrapyCommand): @@ -21,6 +22,11 @@ class Command(ScrapyCommand): def long_desc(self): return "Interactive console for scraping the given url" + def add_options(self, parser): + ScrapyCommand.add_options(self, parser) + parser.add_option("-c", dest="code", + help="evaluate the code in the shell, print the result and exit") + def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be available in the shell @@ -29,6 +35,12 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None - shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True) - shell.start(url=url).addBoth(lambda _: self.crawler.stop()) + shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True, \ + code=opts.code) + def err(f): + log.err(f, "Shell error") + self.exitcode = 1 + d = shell.start(url=url) + d.addErrback(err) + d.addBoth(lambda _: self.crawler.stop()) self.crawler.start() diff --git a/scrapy/shell.py b/scrapy/shell.py index 573a42147..cb6706f86 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -27,12 +27,13 @@ class Shell(object): relevant_classes = (BaseSpider, Request, Response, BaseItem, \ XPathSelector, Settings) - def __init__(self, crawler, update_vars=None, inthread=False): + def __init__(self, crawler, update_vars=None, inthread=False, code=None): self.crawler = crawler self.vars = {} self.update_vars = update_vars or (lambda x: None) self.item_class = load_object(settings['DEFAULT_ITEM_CLASS']) self.inthread = inthread + self.code = code def start(self, *a, **kw): # disable accidental Ctrl-C key press from shutting down the engine @@ -50,7 +51,10 @@ class Shell(object): elif response: request = response.request self.populate_vars(request.url, response, request, spider) - start_python_console(self.vars) + if self.code: + print eval(self.code, globals(), self.vars) + else: + start_python_console(self.vars) def _schedule(self, request, spider): if spider is None: @@ -61,21 +65,16 @@ class Shell(object): return self.crawler.engine.schedule(request, spider) def fetch(self, request_or_url, spider=None): - # we enclose all this code in a try/except block to see errors when - # they happen in a thread - try: - if isinstance(request_or_url, Request): - request = request_or_url - url = request.url - else: - url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - response = None - response = threads.blockingCallFromThread(reactor, \ - self._schedule, request, spider) - self.populate_vars(url, response, request, spider) - except: - log.err(Failure(), "Error fetching: %s" % request_or_url, spider=spider) + if isinstance(request_or_url, Request): + request = request_or_url + url = request.url + else: + url = any_to_uri(request_or_url) + request = Request(url, dont_filter=True) + response = None + response = threads.blockingCallFromThread(reactor, \ + self._schedule, request, spider) + self.populate_vars(url, response, request, spider) def populate_vars(self, url=None, response=None, request=None, spider=None): item = self.item_class() @@ -93,7 +92,8 @@ class Shell(object): self.vars['view'] = open_in_browser self.vars['shelp'] = self.print_help self.update_vars(self.vars) - self.print_help() + if not self.code: + self.print_help() def print_help(self): self.p("Available Scrapy objects:")