Added a synchronous get method which also updates console user namespace.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40179

scrapy/trunk/scrapy/command/commands/scrape.py

@@ -1,43 +1,44 @@
+from twisted.internet import reactor
+
 import scrapy
 from scrapy.command import ScrapyCommand
-from scrapy.fetcher import fetch
 from scrapy.spider import spiders
 from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector
 from scrapy.utils.misc import load_class
 from scrapy.extension import extensions
 from scrapy.conf import settings
+from scrapy.core.manager import scrapymanager
 
-def get_url(url):
-    from scrapy.http import Request
-    from scrapy.core.downloader.handlers import download_any
-    request = Request(url)
-    spider = spiders.fromurl(url)
+#This code comes from twisted 8. We define here while
+#using old twisted version.
+def blockingCallFromThread(reactor, f, *a, **kw):
+    """
+    Run a function in the reactor from a thread, and wait for the result
+    synchronously, i.e. until the callback chain returned by the function
+    get a result.
 
-    return download_any(request, spider)
+    @param reactor: The L{IReactorThreads} provider which will be used to
+        schedule the function call.
+    @param f: the callable to run in the reactor thread
+    @type f: any callable.
+    @param a: the arguments to pass to C{f}.
+    @param kw: the keyword arguments to pass to C{f}.
 
-def load_url(url, response):
-    vars = {}
-    itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
-    item = itemcls()
-    vars['item'] = item
-    vars['xxs'] = XmlXPathSelector(response)
-    vars['hxs'] = HtmlXPathSelector(response)
-    if url:
-        vars['url'] = url
-        vars['response'] = response
-        vars['spider'] = spiders.fromurl(url)
-    vars['get'] = get_url
-    return vars
-
-def print_vars(vars):
-    print '-' * 78
-    print "Available local variables:"
-    for key, val in vars.iteritems():
-        if isinstance(val, basestring):
-            print "   %s: %s" % (key, val)
-        else:
-            print "   %s: %s" % (key, val.__class__)
-    print '-' * 78
+    @return: the result of the callback chain.
+    @raise: any error raised during the callback chain.
+    """
+    import Queue
+    from twisted.python import failure
+    from twisted.internet import defer
+    queue = Queue.Queue()
+    def _callFromThread():
+        result = defer.maybeDeferred(f, *a, **kw)
+        result.addBoth(queue.put)
+    reactor.callFromThread(_callFromThread)
+    result = queue.get()
+    if isinstance(result, failure.Failure):
+        result.raiseException()
+    return result
 
 class Command(ScrapyCommand):
     def syntax(self):
@@ -49,12 +50,66 @@ class Command(ScrapyCommand):
     def long_desc(self):
         return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html"
 
-    def update_vars(self, vars):
+    def update_vars(self):
         """ You can use this function to update the local variables that will be available in the scrape console """
         pass
 
-    def run(self, args, opts):
+    def get_url(self, url):
+        #def _get_callback(_response):
+            #print "done"
+            #if not _response:
+                #if not opts.loglevel:
+                    #print 'Nothing downloaded, run with -o DEBUG to see why it failed'
+                #scrapymanager.stop()
+                #return
+            #self.generate_vars(url, response)
+
+        #def _errback(_failure):
+            #print _failure
+
+        print "Downloading URL...           ",
+        from scrapy.http import Request, Response
+        from scrapy.core.downloader.handlers import download_any
+        from scrapy.fetcher import get_or_create_spider
+        r = Request(url)
+        spider = get_or_create_spider(url)
+        try:
+            result = blockingCallFromThread(reactor, download_any, r, spider)
+            if isinstance(result, Response):
+                print "OK"
+                self.generate_vars(url, result)
+        except Exception, e:
+            print "Error: %s" % e
+            
+    def generate_vars(self, url, response):
+        itemcls = load_class(settings['DEFAULT_ITEM_CLASS'])
+        item = itemcls()
+        self.vars['item'] = item
+        if url:
+            self.vars['xxs'] = XmlXPathSelector(response)
+            self.vars['hxs'] = HtmlXPathSelector(response)
+            self.vars['url'] = url
+            self.vars['response'] = response
+            self.vars['spider'] = spiders.fromurl(url)
+        self.vars['get'] = self.get_url
+        self.vars['scrape_help'] = self.print_vars
+        self.update_vars()
+        self.user_ns.update(self.vars)
+        self.print_vars()
         
+    def print_vars(self):
+        print '-' * 78
+        print "Available local variables:"
+        for key, val in self.vars.iteritems():
+            if isinstance(val, basestring):
+                print "   %s: %s" % (key, val)
+            else:
+                print "   %s: %s" % (key, val.__class__)
+        print '-' * 78
+    
+    def run(self, args, opts):
+        self.vars = {}
+        self.user_ns = {}
         url = None
         if args:
             url = args[0]
@@ -64,32 +119,30 @@ class Command(ScrapyCommand):
         print "Enabling Scrapy extensions...",
         extensions.load()
         print "done"
-        response = None
-        if url:
-            print "Downloading URL...           ",
-            responses = fetch([url])
-            print "done"
-            if not responses:
-                if not opts.loglevel:
-                    print 'Nothing downloaded, run with -o DEBUG to see why it failed'
-                return
-            response = responses[0]
 
-        vars = load_url(url, response)
-        self.update_vars(vars)
-        print_vars(vars)
+        def _console_thread():
 
-        try: # use IPython if available
-            import IPython
-            shell = IPython.Shell.IPShell(argv=[], user_ns=vars)
-            shell.mainloop()
-        except ImportError:
-            import code
-            try: # readline module is only available on unix systems
-                import readline
-            except ImportError:
-                pass
+            if url:
+                self.get_url(url)
             else:
-                import rlcompleter
-                readline.parse_and_bind("tab:complete")
-            code.interact(local=vars)
+                self.generate_vars(url, None)
+            
+            
+            try: # use IPython if available
+                import IPython
+                shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns)
+                shell.mainloop()
+                reactor.callFromThread(scrapymanager.stop)
+            except ImportError:
+                import code
+                try: # readline module is only available on unix systems
+                    import readline
+                except ImportError:
+                    pass
+                else:
+                    import rlcompleter
+                    readline.parse_and_bind("tab:complete")
+                code.interact(local=self.vars)
+
+        reactor.callInThread(_console_thread)
+        scrapymanager.start()