Added support for passing code to evaluate in Scrapy shell command (closes #249) and simplified handling of shell errors

scrapy/commands/shell.py

@@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
 
 from scrapy.command import ScrapyCommand
 from scrapy.shell import Shell
+from scrapy import log
 
 class Command(ScrapyCommand):
 
@@ -21,6 +22,11 @@ class Command(ScrapyCommand):
     def long_desc(self):
         return "Interactive console for scraping the given url"
 
+    def add_options(self, parser):
+        ScrapyCommand.add_options(self, parser)
+        parser.add_option("-c", dest="code",
+            help="evaluate the code in the shell, print the result and exit")
+
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
         available in the shell
@@ -29,6 +35,12 @@ class Command(ScrapyCommand):
 
     def run(self, args, opts):
         url = args[0] if args else None
-        shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True)
-        shell.start(url=url).addBoth(lambda _: self.crawler.stop())
+        shell = Shell(self.crawler, update_vars=self.update_vars, inthread=True, \
+            code=opts.code)
+        def err(f):
+            log.err(f, "Shell error")
+            self.exitcode = 1
+        d = shell.start(url=url)
+        d.addErrback(err)
+        d.addBoth(lambda _: self.crawler.stop())
         self.crawler.start()

scrapy/shell.py

@@ -27,12 +27,13 @@ class Shell(object):
     relevant_classes = (BaseSpider, Request, Response, BaseItem, \
         XPathSelector, Settings)
 
-    def __init__(self, crawler, update_vars=None, inthread=False):
+    def __init__(self, crawler, update_vars=None, inthread=False, code=None):
         self.crawler = crawler
         self.vars = {}
         self.update_vars = update_vars or (lambda x: None)
         self.item_class = load_object(settings['DEFAULT_ITEM_CLASS'])
         self.inthread = inthread
+        self.code = code
 
     def start(self, *a, **kw):
         # disable accidental Ctrl-C key press from shutting down the engine
@@ -50,7 +51,10 @@ class Shell(object):
         elif response:
             request = response.request
             self.populate_vars(request.url, response, request, spider)
-        start_python_console(self.vars)
+        if self.code:
+            print eval(self.code, globals(), self.vars)
+        else:
+            start_python_console(self.vars)
 
     def _schedule(self, request, spider):
         if spider is None:
@@ -61,21 +65,16 @@ class Shell(object):
         return self.crawler.engine.schedule(request, spider)
 
     def fetch(self, request_or_url, spider=None):
-        # we enclose all this code in a try/except block to see errors when
-        # they happen in a thread
-        try:
-            if isinstance(request_or_url, Request):
-                request = request_or_url
-                url = request.url
-            else:
-                url = any_to_uri(request_or_url)
-                request = Request(url, dont_filter=True)
-            response = None
-            response = threads.blockingCallFromThread(reactor, \
-                self._schedule, request, spider)
-            self.populate_vars(url, response, request, spider)
-        except:
-            log.err(Failure(), "Error fetching: %s" % request_or_url, spider=spider)
+        if isinstance(request_or_url, Request):
+            request = request_or_url
+            url = request.url
+        else:
+            url = any_to_uri(request_or_url)
+            request = Request(url, dont_filter=True)
+        response = None
+        response = threads.blockingCallFromThread(reactor, \
+            self._schedule, request, spider)
+        self.populate_vars(url, response, request, spider)
 
     def populate_vars(self, url=None, response=None, request=None, spider=None):
         item = self.item_class()
@@ -93,7 +92,8 @@ class Shell(object):
         self.vars['view'] = open_in_browser
         self.vars['shelp'] = self.print_help
         self.update_vars(self.vars)
-        self.print_help()
+        if not self.code:
+            self.print_help()
 
     def print_help(self):
         self.p("Available Scrapy objects:")