diff --git a/scrapy/trunk/scrapy/command/commands/scrape.py b/scrapy/trunk/scrapy/command/commands/scrape.py index 5809f6851..7224f3d93 100644 --- a/scrapy/trunk/scrapy/command/commands/scrape.py +++ b/scrapy/trunk/scrapy/command/commands/scrape.py @@ -1,43 +1,44 @@ +from twisted.internet import reactor + import scrapy from scrapy.command import ScrapyCommand -from scrapy.fetcher import fetch from scrapy.spider import spiders from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector from scrapy.utils.misc import load_class from scrapy.extension import extensions from scrapy.conf import settings +from scrapy.core.manager import scrapymanager -def get_url(url): - from scrapy.http import Request - from scrapy.core.downloader.handlers import download_any - request = Request(url) - spider = spiders.fromurl(url) +#This code comes from twisted 8. We define here while +#using old twisted version. +def blockingCallFromThread(reactor, f, *a, **kw): + """ + Run a function in the reactor from a thread, and wait for the result + synchronously, i.e. until the callback chain returned by the function + get a result. - return download_any(request, spider) + @param reactor: The L{IReactorThreads} provider which will be used to + schedule the function call. + @param f: the callable to run in the reactor thread + @type f: any callable. + @param a: the arguments to pass to C{f}. + @param kw: the keyword arguments to pass to C{f}. -def load_url(url, response): - vars = {} - itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) - item = itemcls() - vars['item'] = item - vars['xxs'] = XmlXPathSelector(response) - vars['hxs'] = HtmlXPathSelector(response) - if url: - vars['url'] = url - vars['response'] = response - vars['spider'] = spiders.fromurl(url) - vars['get'] = get_url - return vars - -def print_vars(vars): - print '-' * 78 - print "Available local variables:" - for key, val in vars.iteritems(): - if isinstance(val, basestring): - print " %s: %s" % (key, val) - else: - print " %s: %s" % (key, val.__class__) - print '-' * 78 + @return: the result of the callback chain. + @raise: any error raised during the callback chain. + """ + import Queue + from twisted.python import failure + from twisted.internet import defer + queue = Queue.Queue() + def _callFromThread(): + result = defer.maybeDeferred(f, *a, **kw) + result.addBoth(queue.put) + reactor.callFromThread(_callFromThread) + result = queue.get() + if isinstance(result, failure.Failure): + result.raiseException() + return result class Command(ScrapyCommand): def syntax(self): @@ -49,12 +50,66 @@ class Command(ScrapyCommand): def long_desc(self): return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html" - def update_vars(self, vars): + def update_vars(self): """ You can use this function to update the local variables that will be available in the scrape console """ pass - def run(self, args, opts): + def get_url(self, url): + #def _get_callback(_response): + #print "done" + #if not _response: + #if not opts.loglevel: + #print 'Nothing downloaded, run with -o DEBUG to see why it failed' + #scrapymanager.stop() + #return + #self.generate_vars(url, response) + + #def _errback(_failure): + #print _failure + + print "Downloading URL... ", + from scrapy.http import Request, Response + from scrapy.core.downloader.handlers import download_any + from scrapy.fetcher import get_or_create_spider + r = Request(url) + spider = get_or_create_spider(url) + try: + result = blockingCallFromThread(reactor, download_any, r, spider) + if isinstance(result, Response): + print "OK" + self.generate_vars(url, result) + except Exception, e: + print "Error: %s" % e + + def generate_vars(self, url, response): + itemcls = load_class(settings['DEFAULT_ITEM_CLASS']) + item = itemcls() + self.vars['item'] = item + if url: + self.vars['xxs'] = XmlXPathSelector(response) + self.vars['hxs'] = HtmlXPathSelector(response) + self.vars['url'] = url + self.vars['response'] = response + self.vars['spider'] = spiders.fromurl(url) + self.vars['get'] = self.get_url + self.vars['scrape_help'] = self.print_vars + self.update_vars() + self.user_ns.update(self.vars) + self.print_vars() + def print_vars(self): + print '-' * 78 + print "Available local variables:" + for key, val in self.vars.iteritems(): + if isinstance(val, basestring): + print " %s: %s" % (key, val) + else: + print " %s: %s" % (key, val.__class__) + print '-' * 78 + + def run(self, args, opts): + self.vars = {} + self.user_ns = {} url = None if args: url = args[0] @@ -64,32 +119,30 @@ class Command(ScrapyCommand): print "Enabling Scrapy extensions...", extensions.load() print "done" - response = None - if url: - print "Downloading URL... ", - responses = fetch([url]) - print "done" - if not responses: - if not opts.loglevel: - print 'Nothing downloaded, run with -o DEBUG to see why it failed' - return - response = responses[0] - vars = load_url(url, response) - self.update_vars(vars) - print_vars(vars) + def _console_thread(): - try: # use IPython if available - import IPython - shell = IPython.Shell.IPShell(argv=[], user_ns=vars) - shell.mainloop() - except ImportError: - import code - try: # readline module is only available on unix systems - import readline - except ImportError: - pass + if url: + self.get_url(url) else: - import rlcompleter - readline.parse_and_bind("tab:complete") - code.interact(local=vars) + self.generate_vars(url, None) + + + try: # use IPython if available + import IPython + shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns) + shell.mainloop() + reactor.callFromThread(scrapymanager.stop) + except ImportError: + import code + try: # readline module is only available on unix systems + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind("tab:complete") + code.interact(local=self.vars) + + reactor.callInThread(_console_thread) + scrapymanager.start() \ No newline at end of file