From 7858244dca17513cf9995519997697a01a51694b Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 20 Aug 2010 01:33:02 -0300 Subject: [PATCH] Scrapy shell: moved python console starting code to scrapy.utils.console and get rid of noisy console banners --- docs/intro/tutorial.rst | 38 +++++++++++++++----------------------- docs/topics/shell.rst | 36 +++++++++++------------------------- scrapy/shell.py | 27 ++++----------------------- scrapy/utils/console.py | 27 +++++++++++++++++++++++++++ 4 files changed, 57 insertions(+), 71 deletions(-) create mode 100644 scrapy/utils/console.py diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst index 8e95e0f07..b6ff3448f 100644 --- a/docs/intro/tutorial.rst +++ b/docs/intro/tutorial.rst @@ -248,32 +248,24 @@ To start a shell you must go to the project's top level directory and run:: This is what the shell looks like:: - [-] Log opened. - Welcome to Scrapy shell! - Fetching ... + [ ... Scrapy log here ... ] - ------------------------------------------------------------------------------ - Available Scrapy variables: - xxs: - url: http://www.dmoz.org/Computers/Programming/Languages/Python/Books/ - spider: - hxs: - item: - response: - Available commands: - get [url]: Fetch a new URL or re-fetch current Request - shelp: Prints this help. - ------------------------------------------------------------------------------ - Python 2.6.1 (r261:67515, Dec 7 2008, 08:27:41) - Type "copyright", "credits" or "license" for more information. + Available objects: + 2010-08-19 21:45:59-0300 [default] INFO: Spider closed (finished) + xxs + url http://www.dmoz.org/Computers/Programming/Languages/Python/Books/ + request + spider + response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/> + hxs + item Item() - IPython 0.9.1 -- An enhanced Interactive Python. - ? -> Introduction and overview of IPython's features. - %quickref -> Quick reference. - help -> Python's own help system. - object? -> Details about 'object'. ?object also works, ?? prints more. + Convenient shortcuts: + shelp() Print this help + fetch(req_or_url) Fetch a new request or URL and update shell objects + view(response) View response in a browser - In [1]: + In [1]: After the shell loads, you will have the response fetched in a local ``response`` variable, so if you type ``response.body`` you will see the body diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 213a879c1..b6ce20d2a 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -113,25 +113,18 @@ list of available objects and some help:: Fetching ... Available objects - ================= - - xxs : - url : http://scrapy.org - request : - spider : - hxs : - item : Item() - response : + xxs + url http://scrapy.org + request + spider + hxs + item Item() + response Available shortcuts - =================== - - shelp() : Prints this help. - fetch(req_or_url) : Fetch a new request or URL and update objects - view(response) : View response in a browser - - Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18) - Type "help", "copyright", "credits" or "license" for more information. + shelp() Prints this help. + fetch(req_or_url) Fetch a new request or URL and update objects + view(response) View response in a browser >>> @@ -177,16 +170,9 @@ When you the spider you will get something similar to this:: 2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled (referer: ) 2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled (referer: ) - Scrapy Shell - ============ - - Inspecting: >> response.url 'http://www.example.com/products.php' diff --git a/scrapy/shell.py b/scrapy/shell.py index dfd5938e8..5b9ddb54d 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -17,6 +17,7 @@ from scrapy.spider import BaseSpider, spiders from scrapy.selector import XmlXPathSelector, HtmlXPathSelector from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser +from scrapy.utils.console import start_python_console from scrapy.conf import settings from scrapy.core.manager import scrapymanager from scrapy.core.queue import KeepAliveExecutionQueue @@ -24,7 +25,7 @@ from scrapy.http import Request, TextResponse def relevant_var(varname): return varname not in ['shelp', 'fetch', 'view', '__builtins__', 'In', \ - 'Out', 'help'] and not varname.startswith('_') + 'Out', 'help', 'namespace'] and not varname.startswith('_') def parse_url(url): """Parse url which can be a direct path to a direct file""" @@ -120,27 +121,7 @@ class Shell(object): request = response.request url = request.url self.populate_vars(url, response, request) - self._run_console() - - def _run_console(self): - try: - try: # use IPython if available - import IPython - shell = IPython.Shell.IPShell(argv=[], user_ns=self.vars) - ip = shell.IP.getapi() - shell.mainloop() - except ImportError: - import code - try: # readline module is only available on unix systems - import readline - except ImportError: - pass - else: - import rlcompleter - readline.parse_and_bind("tab:complete") - code.interact(local=self.vars) - except SystemExit: # raised when using exit() in python code.interact - pass + start_python_console(self.vars) def _console_thread(self, url=None): self.populate_vars() @@ -148,7 +129,7 @@ class Shell(object): result = self.fetch(url, print_help=True) else: self.print_help() - self._run_console() + start_python_console(self.vars) reactor.callFromThread(scrapymanager.stop) def inspect_response(response): diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py new file mode 100644 index 000000000..9edf36e39 --- /dev/null +++ b/scrapy/utils/console.py @@ -0,0 +1,27 @@ + +def start_python_console(namespace=None, noipython=False): + """Start Python console binded to the given namespace. If IPython is + available, an IPython console will be started instead, unless `noipython` + is True. Also, tab completion will be used on Unix systems. + """ + if namespace is None: + namespace = {} + try: + try: # use IPython if available + if noipython: + raise ImportError + import IPython + shell = IPython.Shell.IPShellEmbed(argv=[], user_ns=namespace) + shell() + except ImportError: + import code + try: # readline module is only available on unix systems + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind("tab:complete") + code.interact(banner='', local=namespace) + except SystemExit: # raised when using exit() in python code.interact + pass