diff --git a/scrapy/trunk/docs/topics/shell.rst b/scrapy/trunk/docs/topics/shell.rst index c9f2d6476..19ec83259 100644 --- a/scrapy/trunk/docs/topics/shell.rst +++ b/scrapy/trunk/docs/topics/shell.rst @@ -90,9 +90,9 @@ Example of shell session Here's an example of a typical shell session where we start by scraping the http://scrapy.org page, and then the http://slashdot.org page. Note, however, -that the data extracted may not be the same when you try this as those pages +that the data extracted here may not be the same when you try, as those pages are not static and could have changed by the time you test this. The purpose of -this example is only to get you familiarized with how the Scrapy shell works. +this example is just to get you familiarized with how the Scrapy shell works. :: @@ -104,12 +104,12 @@ this example is only to get you familiarized with how the Scrapy shell works. ------------------------------------------------------------ Available Scrapy variables: - xxs: + xxs: url: http://scrapy.org spider: - hxs: + hxs: item: - response: + response: Available commands: get : Fetches an url and updates all variables. scrapehelp: Prints this help. @@ -130,13 +130,12 @@ this example is only to get you familiarized with how the Scrapy shell works. Downloading URL... Done. ------------------------------------------------------------ Available Scrapy variables: - xxs: + xxs: url: http://slashdot.org spider: - hxs: + hxs: item: - r: - response: + response: Available commands: get : Fetches an url and updates all variables. scrapehelp: Prints this help. diff --git a/scrapy/trunk/scrapy/command/commands/shell.py b/scrapy/trunk/scrapy/command/commands/shell.py index 9bc5d2a93..c1c9ae53c 100644 --- a/scrapy/trunk/scrapy/command/commands/shell.py +++ b/scrapy/trunk/scrapy/command/commands/shell.py @@ -1,49 +1,19 @@ -from twisted.internet import reactor +import os +import urllib +import urlparse + +from twisted.internet import reactor, threads -import os, urllib, urlparse -import scrapy from scrapy.command import ScrapyCommand from scrapy.spider import spiders from scrapy.xpath import XmlXPathSelector, HtmlXPathSelector from scrapy.utils.misc import load_object -from scrapy.extension import extensions from scrapy.conf import settings from scrapy.core.manager import scrapymanager from scrapy.core.engine import scrapyengine from scrapy.http import Request from scrapy.fetcher import get_or_create_spider -#This code comes from twisted 8. We define here while -#using old twisted version. -def blockingCallFromThread(reactor, f, *a, **kw): - """ - Run a function in the reactor from a thread, and wait for the result - synchronously, i.e. until the callback chain returned by the function - get a result. - - @param reactor: The L{IReactorThreads} provider which will be used to - schedule the function call. - @param f: the callable to run in the reactor thread - @type f: any callable. - @param a: the arguments to pass to C{f}. - @param kw: the keyword arguments to pass to C{f}. - - @return: the result of the callback chain. - @raise: any error raised during the callback chain. - """ - import Queue - from twisted.python import failure - from twisted.internet import defer - queue = Queue.Queue() - def _callFromThread(): - result = defer.maybeDeferred(f, *a, **kw) - result.addBoth(queue.put) - reactor.callFromThread(_callFromThread) - result = queue.get() - if isinstance(result, failure.Failure): - result.raiseException() - return result - class Command(ScrapyCommand): def syntax(self): return "[url]" @@ -55,7 +25,7 @@ class Command(ScrapyCommand): return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html" def update_vars(self): - """ You can use this function to update the local variables that will be available in the scrape console """ + """ You can use this function to update the Scrapy objects that will be available in the shell""" pass def get_url(self, url): @@ -74,12 +44,11 @@ class Command(ScrapyCommand): self.result = response return [] - print "Downloading URL... ", + print "Downloading URL..." r = Request(url, callback=_get_response) spider = get_or_create_spider(url) - blockingCallFromThread(reactor, scrapyengine.crawl, r, spider) + threads.blockingCallFromThread(reactor, scrapyengine.crawl, r, spider) if self.result: - print "Done." self.result.request = r self.generate_vars(url, self.result) return True @@ -99,17 +68,14 @@ class Command(ScrapyCommand): self.print_vars() def print_vars(self): - print '-' * 78 - print "Available local variables:" + print '-' * 60 + print "Available Scrapy objects:" for key, val in self.vars.iteritems(): - if isinstance(val, basestring): - print " %s: %s" % (key, val) - else: - print " %s: %s" % (key, val.__class__) + print " %s: %s" % (key, val) print "Available commands:" - print " get : Fetches an url and updates all variables." - print " scrapehelp: Prints this help." - print '-' * 78 + print " get : Fetches a new page and updates all Scrapy objects." + print " shelp: Prints this help." + print '-' * 60 def run(self, args, opts): self.vars = {} @@ -118,7 +84,7 @@ class Command(ScrapyCommand): if args: url = args[0] - print "Scrapy %s - Interactive scraping console\n" % scrapy.__version__ + print "Welcome to Scrapy shell!" def _console_thread(): @@ -138,7 +104,7 @@ class Command(ScrapyCommand): shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns) ip = shell.IP.getapi() ip.expose_magic("get", _get_magic) - ip.expose_magic("scrapehelp", _help_magic) + ip.expose_magic("shelp", _help_magic) shell.mainloop() reactor.callFromThread(scrapymanager.stop) except ImportError: diff --git a/scrapy/trunk/scrapy/patches/monkeypatches.py b/scrapy/trunk/scrapy/patches/monkeypatches.py index 6425ea27c..a9a0890ce 100644 --- a/scrapy/trunk/scrapy/patches/monkeypatches.py +++ b/scrapy/trunk/scrapy/patches/monkeypatches.py @@ -1,18 +1,13 @@ """ -Monkey patches +Monkey patches for supporting Twisted 2.5.0 +""" -These are generally a bad idea. -""" import twisted - -# Extend limit for BeautifulSoup parsing loops -#import sys -#sys.setrecursionlimit(7400) - def apply_patches(): if twisted.__version__ < '8.0.0': patch_HTTPPageGetter_handleResponse() + add_missing_blockingCallFromThread() # bugfix not present in twisted 2.5 for handling empty response of HEAD requests @@ -43,3 +38,38 @@ def patch_HTTPPageGetter_handleResponse(): # stupid... self.transport.loseConnection() setattr(HTTPPageGetter, 'handleResponse', _handleResponse) + +# This function comes bundled with Twisted 8.x and above +def add_missing_blockingCallFromThread(): + import Queue + from twisted.internet import defer + from twisted.python import failure + + def blockingCallFromThread(reactor, f, *a, **kw): + """ + Run a function in the reactor from a thread, and wait for the result + synchronously, i.e. until the callback chain returned by the function + get a result. + + @param reactor: The L{IReactorThreads} provider which will be used to + schedule the function call. + @param f: the callable to run in the reactor thread + @type f: any callable. + @param a: the arguments to pass to C{f}. + @param kw: the keyword arguments to pass to C{f}. + + @return: the result of the callback chain. + @raise: any error raised during the callback chain. + """ + queue = Queue.Queue() + def _callFromThread(): + result = defer.maybeDeferred(f, *a, **kw) + result.addBoth(queue.put) + reactor.callFromThread(_callFromThread) + result = queue.get() + if isinstance(result, failure.Failure): + result.raiseException() + return result + + from twisted.internet import threads + threads.blockingCallFromThread = blockingCallFromThread