mirror of https://github.com/scrapy/scrapy.git
Scrapy shell: moved python console starting code to scrapy.utils.console and get rid of noisy console banners
This commit is contained in:
Pablo Hoffman
parent
136b0e7450
commit
7858244dca
|
|
@ -248,32 +248,24 @@ To start a shell you must go to the project's top level directory and run::
|
|||
|
||||
This is what the shell looks like::
|
||||
|
||||
[-] Log opened.
|
||||
Welcome to Scrapy shell!
|
||||
Fetching <http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>...
|
||||
[ ... Scrapy log here ... ]
|
||||
|
||||
------------------------------------------------------------------------------
|
||||
Available Scrapy variables:
|
||||
xxs: <class 'scrapy.selector.XmlXPathSelector'>
|
||||
url: http://www.dmoz.org/Computers/Programming/Languages/Python/Books/
|
||||
spider: <class 'dmoz.spiders.dmoz.OpenDirectorySpider'>
|
||||
hxs: <class 'scrapy.selector.HtmlXPathSelector'>
|
||||
item: <class 'scrapy.item.Item'>
|
||||
response: <class 'scrapy.http.response.html.HtmlResponse'>
|
||||
Available commands:
|
||||
get [url]: Fetch a new URL or re-fetch current Request
|
||||
shelp: Prints this help.
|
||||
------------------------------------------------------------------------------
|
||||
Python 2.6.1 (r261:67515, Dec 7 2008, 08:27:41)
|
||||
Type "copyright", "credits" or "license" for more information.
|
||||
Available objects:
|
||||
2010-08-19 21:45:59-0300 [default] INFO: Spider closed (finished)
|
||||
xxs <XmlXPathSelector (http://www.dmoz.org/Computers/Programming/Languages/Python/Books/) xpath=None>
|
||||
url http://www.dmoz.org/Computers/Programming/Languages/Python/Books/
|
||||
request <GET http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
spider <BaseSpider 'default' at 0x1b6c2d0>
|
||||
response <200 http://www.dmoz.org/Computers/Programming/Languages/Python/Books/>
|
||||
hxs <HtmlXPathSelector (http://www.dmoz.org/Computers/Programming/Languages/Python/Books/) xpath=None>
|
||||
item Item()
|
||||
|
||||
IPython 0.9.1 -- An enhanced Interactive Python.
|
||||
? -> Introduction and overview of IPython's features.
|
||||
%quickref -> Quick reference.
|
||||
help -> Python's own help system.
|
||||
object? -> Details about 'object'. ?object also works, ?? prints more.
|
||||
Convenient shortcuts:
|
||||
shelp() Print this help
|
||||
fetch(req_or_url) Fetch a new request or URL and update shell objects
|
||||
view(response) View response in a browser
|
||||
|
||||
In [1]:
|
||||
In [1]:
|
||||
|
||||
After the shell loads, you will have the response fetched in a local
|
||||
``response`` variable, so if you type ``response.body`` you will see the body
|
||||
|
|
|
|||
|
|
@ -113,25 +113,18 @@ list of available objects and some help::
|
|||
|
||||
Fetching <http://scrapy.org>...
|
||||
Available objects
|
||||
=================
|
||||
|
||||
xxs : <XmlXPathSelector (http://scrapy.org) xpath=None>
|
||||
url : http://scrapy.org
|
||||
request : <http://scrapy.org>
|
||||
spider : <scrapy.spider.models.BaseSpider object at 0x2bed9d0>
|
||||
hxs : <HtmlXPathSelector (http://scrapy.org) xpath=None>
|
||||
item : Item()
|
||||
response : <http://scrapy.org>
|
||||
xxs <XmlXPathSelector (http://scrapy.org) xpath=None>
|
||||
url http://scrapy.org
|
||||
request <http://scrapy.org>
|
||||
spider <scrapy.spider.models.BaseSpider object at 0x2bed9d0>
|
||||
hxs <HtmlXPathSelector (http://scrapy.org) xpath=None>
|
||||
item Item()
|
||||
response <http://scrapy.org>
|
||||
|
||||
Available shortcuts
|
||||
===================
|
||||
|
||||
shelp() : Prints this help.
|
||||
fetch(req_or_url) : Fetch a new request or URL and update objects
|
||||
view(response) : View response in a browser
|
||||
|
||||
Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18)
|
||||
Type "help", "copyright", "credits" or "license" for more information.
|
||||
shelp() Prints this help.
|
||||
fetch(req_or_url) Fetch a new request or URL and update objects
|
||||
view(response) View response in a browser
|
||||
|
||||
>>>
|
||||
|
||||
|
|
@ -177,16 +170,9 @@ When you the spider you will get something similar to this::
|
|||
2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled <http://www.example.com/> (referer: <None>)
|
||||
2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled <http://www.example.com/products.php> (referer: <http://www.example.com/>)
|
||||
|
||||
Scrapy Shell
|
||||
============
|
||||
|
||||
Inspecting: <http://www.example.com/products.php
|
||||
Scrapy Shell - inspecting response: <http://www.example.com/products.php
|
||||
Use shelp() to see available objects
|
||||
|
||||
Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18)
|
||||
[GCC 4.3.3] on linux2
|
||||
Type "help", "copyright", "credits" or "license" for more information.
|
||||
(InteractiveConsole)
|
||||
>>> response.url
|
||||
'http://www.example.com/products.php'
|
||||
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from scrapy.spider import BaseSpider, spiders
|
|||
from scrapy.selector import XmlXPathSelector, HtmlXPathSelector
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.response import open_in_browser
|
||||
from scrapy.utils.console import start_python_console
|
||||
from scrapy.conf import settings
|
||||
from scrapy.core.manager import scrapymanager
|
||||
from scrapy.core.queue import KeepAliveExecutionQueue
|
||||
|
|
@ -24,7 +25,7 @@ from scrapy.http import Request, TextResponse
|
|||
|
||||
def relevant_var(varname):
|
||||
return varname not in ['shelp', 'fetch', 'view', '__builtins__', 'In', \
|
||||
'Out', 'help'] and not varname.startswith('_')
|
||||
'Out', 'help', 'namespace'] and not varname.startswith('_')
|
||||
|
||||
def parse_url(url):
|
||||
"""Parse url which can be a direct path to a direct file"""
|
||||
|
|
@ -120,27 +121,7 @@ class Shell(object):
|
|||
request = response.request
|
||||
url = request.url
|
||||
self.populate_vars(url, response, request)
|
||||
self._run_console()
|
||||
|
||||
def _run_console(self):
|
||||
try:
|
||||
try: # use IPython if available
|
||||
import IPython
|
||||
shell = IPython.Shell.IPShell(argv=[], user_ns=self.vars)
|
||||
ip = shell.IP.getapi()
|
||||
shell.mainloop()
|
||||
except ImportError:
|
||||
import code
|
||||
try: # readline module is only available on unix systems
|
||||
import readline
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
import rlcompleter
|
||||
readline.parse_and_bind("tab:complete")
|
||||
code.interact(local=self.vars)
|
||||
except SystemExit: # raised when using exit() in python code.interact
|
||||
pass
|
||||
start_python_console(self.vars)
|
||||
|
||||
def _console_thread(self, url=None):
|
||||
self.populate_vars()
|
||||
|
|
@ -148,7 +129,7 @@ class Shell(object):
|
|||
result = self.fetch(url, print_help=True)
|
||||
else:
|
||||
self.print_help()
|
||||
self._run_console()
|
||||
start_python_console(self.vars)
|
||||
reactor.callFromThread(scrapymanager.stop)
|
||||
|
||||
def inspect_response(response):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
|
||||
def start_python_console(namespace=None, noipython=False):
|
||||
"""Start Python console binded to the given namespace. If IPython is
|
||||
available, an IPython console will be started instead, unless `noipython`
|
||||
is True. Also, tab completion will be used on Unix systems.
|
||||
"""
|
||||
if namespace is None:
|
||||
namespace = {}
|
||||
try:
|
||||
try: # use IPython if available
|
||||
if noipython:
|
||||
raise ImportError
|
||||
import IPython
|
||||
shell = IPython.Shell.IPShellEmbed(argv=[], user_ns=namespace)
|
||||
shell()
|
||||
except ImportError:
|
||||
import code
|
||||
try: # readline module is only available on unix systems
|
||||
import readline
|
||||
except ImportError:
|
||||
pass
|
||||
else:
|
||||
import rlcompleter
|
||||
readline.parse_and_bind("tab:complete")
|
||||
code.interact(banner='', local=namespace)
|
||||
except SystemExit: # raised when using exit() in python code.interact
|
||||
pass
|
||||
Loading…
Reference in New Issue