From ea4f16bae6002145faa94a7d6603187faef8bc3d Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 27 Aug 2009 18:24:08 -0300 Subject: [PATCH] refactored scrapy shell implementation, dropping IPython dependency, and adding a new 'view' shortcut --- docs/index.rst | 6 +- docs/topics/shell.rst | 180 +++++++++++++------------------ scrapy/command/commands/shell.py | 117 ++------------------ scrapy/shell.py | 127 ++++++++++++++++++++++ 4 files changed, 217 insertions(+), 213 deletions(-) create mode 100644 scrapy/shell.py diff --git a/docs/index.rst b/docs/index.rst index d0c6be63f..6363c6024 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -67,12 +67,12 @@ Scraping basics :ref:`topics-selectors` Extract the data from web pages. +:ref:`topics-shell` + Test your extraction code in an interactive environment. + :ref:`topics-loaders` Populate your items with the extracted data. -:ref:`topics-shell` - Test your extracted data in an interactive environment. - :ref:`topics-item-pipeline` Post-process and store your scraped data. diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 3dea2bbe0..f3b40c25c 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -1,8 +1,8 @@ .. _topics-shell: -================ -The Scrapy shell -================ +============ +Scrapy shell +============ The Scrapy shell is an interactive shell where you can try and debug your scraping code very quickly, without having to run the spider. It's meant to be @@ -17,12 +17,13 @@ to run the spider to test every change. Once you get familiarized with the Scrapy shell you'll see that it's an invaluable tool for developing and debugging your spiders. -Requirements -============ +If you have `IPython`_ installed, the Scrapy shell will use it (instead of the +standard Python console). The `IPython`_ console is a much more powerful and +provides smart auto-completion and colorized output, among other things. -The Scrapy shell is powered by `IPython`_, so you need to install it before you -can use it, but we highly recommend to do it. See the `IPython installation -guide`_ for more info. +We highly recommend you to install `IPython`_, specially if you're working on +Unix systems (where `IPython`_ excels). See the `IPython installation guide`_ +for more info. .. _IPython: http://ipython.scipy.org/ .. _IPython installation guide: http://ipython.scipy.org/doc/rel-0.9.1/html/install/index.html @@ -34,141 +35,114 @@ To launch the shell type:: scrapy-ctl.py shell -Where the ```` is the URL you want to screen scrape. +Where the ```` is the URL you want to scrape. Using the shell =============== -The Scrapy shell is just a regular `IPython`_ shell (see `IPython -documentation`_) with a few extensions: +The Scrapy shell is just a regular Python console (or `IPython` shell if you +have it available) which provides some additional functions available by +default (as shortcuts): -.. _IPython documentation: http://ipython.scipy.org/moin/Documentation +Built-in Shortcuts +------------------ -Custom Shell Commands ---------------------- + * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``%shelp`` - show the status of your Scrapy objects and get a list of - all (Scrapy related) available objects. + * ``fetch(request_or_url)`` - fetch a new response from the given request or + URL and update all related objects accordingly. - * ``%get [url]``- fetch a new response from the given URL and update all - Scrapy objects accordingly. If the url is omitted, the last request is - re-fetched. Since Requests are mutable objects, you can modify the Request - "in place" and issue ``get`` to fetch it again with the modifications you've - made. + * ``view(response)`` - open the given response in your local web browser, for + inspection. Note that this will generate a temporary file which won't be + removed automatically. -These commands can be typed without the leading percent sign if you have -`automagic commands`_ enabled in IPython. +Built-in Objects +---------------- -.. _automagic commands: http://ipython.scipy.org/doc/manual/html/interactive/reference.html#magic-command-system - -Custom Shell Objects --------------------- - -The console automatically creates some useful Scrapy objects for the downloaded -page, like the :class:`~scrapy.http.Response` object and the -:class:`~scrapy.selector.XPathSelector` objects (for both HTML and XML content). +The Scrapy shell automatically creates some convenient objects from the +downloaded page, like the :class:`~scrapy.http.Response` object and the +:class:`~scrapy.selector.XPathSelector` objects (for both HTML and XML +content). Those objects are: * ``url`` - the URL being analyzed - * ``spider`` - the Spider which is configured to process the URL to analyze, - or a :class:`~scrapy.spider.BaseSpider` object if there is not spider - configured to process that URL + * ``spider`` - the Spider which is known to handle the URL, or a + :class:`~scrapy.spider.BaseSpider` object if there is no spider is found for + the current URL * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched - page. You can modify this Request "in place" and re-fetch it using the - command ``get`` without no argument. + page. You can modify this request using :meth:`~scrapy.http.Request.replace` + fetch a new request (without leaving the shell) using the ``fetch`` + shortcut. - * ``response`` - a :class:`~scrapy.http.Response` object of the last fetched - page + * ``response`` - a :class:`~scrapy.http.Response` object contaning the last + fetched page - * ``hxs`` - a :class:`~scrapy.selector.HtmlXPathSelector` object for the Response - of the downloaded page - - * ``xxs`` - a :class:`~scrapy.selector.XmlXPathSelector` object for the Response - of the downloaded page - - * ``get ``- download a new response from the given URL and update all - Scrapy objects accordingly + * ``hxs`` - a :class:`~scrapy.selector.HtmlXPathSelector` object constructed + with the last response fetched + * ``xxs`` - a :class:`~scrapy.selector.XmlXPathSelector` object constructed + with the last response fetched Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the http://scrapy.org page, and then proceed to scrape the http://slashdot.org -page. Finally, we modify the (slashdot) request method to POST and re-fetch it +page. Finally, we modify the (Slashdot) request method to POST and re-fetch it getting a HTTP 405 (method not allowed) error. We end the session by typing -Ctrl-D (in the ``In [6]`` prompt). +Ctrl-D (in Unix systems) or Ctrl-Z in Windows. Keep in mind that the data extracted here may not be the same when you try it, as those pages are not static and could have changed by the time you test this. The only purpose of this example is to get you familiarized with how the Scrapy shell works. -:: +First, we launch the shell:: - python scrapy-ctl.py shell http://scrapy.org + python scrapy-ctl.py shell http://scrapy.org --nolog + +Then, the shell fetches the url (using the Scrapy downloader) and prints the +list of available objects and some help:: - 2009-04-02 16:56:22-0300 [-] Log opened. - Welcome to Scrapy shell! Fetching ... + Available objects + ================= - ------------------------------------------------------------ - Available Scrapy variables: - xxs: - url: http://scrapy.org - request: - spider: - hxs: - item: - response: - Available commands: - get [url]: Fetch a new URL or re-fetch current Request - shelp: Prints this help. - ------------------------------------------------------------ - Python 2.5.2 (r252:60911, Oct 5 2008, 19:29:17) - Type "copyright", "credits" or "license" for more information. + xxs : + url : http://scrapy.org + request : + spider : + hxs : + item : Item() + response : - IPython 0.8.4 -- An enhanced Interactive Python. - ? -> Introduction and overview of IPython's features. - %quickref -> Quick reference. - help -> Python's own help system. - object? -> Details about 'object'. ?object also works, ?? prints more. + Available shortcuts + =================== - In [1]: hxs.select("//h2/text()").extract()[2] - Out[1]: u'Welcome to Scrapy' + shelp() : Prints this help. + fetch(req_or_url) : Fetch a new request or URL and update objects + view(response) : View response in a browser - In [2]: get http://slashdot.org + Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18) + Type "help", "copyright", "credits" or "license" for more information. + + >>> + +After that, we can stary playing with the objects:: + + >>> hxs.select("//h2/text()").extract()[0] + u'Welcome to Scrapy' + >>> fetch("http://slashdot.org") Fetching ... - ------------------------------------------------------------ - Available Scrapy variables: - xxs: - url: http://slashdot.org - request: - spider: - hxs: - item: - response: - Available commands: - get : Fetches an url and updates all variables. - scrapehelp: Prints this help. - ------------------------------------------------------------ - - In [3]: hxs.select("//h2/text()").extract() - Out[3]: [u'News for nerds, stuff that matters'] - - In [3]: hxs.select("//h2/text()").extract() - Out[3]: [u'News for nerds, stuff that matters'] - - In [4]: request.method = "POST" - - In [5]: get + Done - use shelp() to see available objects + >>> hxs.select("//h2/text()").extract() + [u'News for nerds, stuff that matters'] + >>> request = request.replace(method="POST") + >>> fetch(request) Fetching ... - 2009-04-03 00:57:39-0300 [decobot/None] ERROR: Downloading from : 405 Method Not Allowed - - In [6]: - 2009-04-03 01:07:12-0300 [-] Main loop terminated. - + 2009-04-03 00:57:39-0300 [scrapybot] ERROR: Downloading from : 405 Method Not Allowed + >>> diff --git a/scrapy/command/commands/shell.py b/scrapy/command/commands/shell.py index 20d3cc8cf..3f8c9e241 100644 --- a/scrapy/command/commands/shell.py +++ b/scrapy/command/commands/shell.py @@ -4,21 +4,8 @@ Scrapy Shell See documentation in docs/topics/shell.rst """ -import os -import urllib -import urlparse - -from twisted.internet import reactor, threads - from scrapy.command import ScrapyCommand -from scrapy.spider import spiders -from scrapy.selector import XmlXPathSelector, HtmlXPathSelector -from scrapy.utils.misc import load_object -from scrapy.conf import settings -from scrapy.core.manager import scrapymanager -from scrapy.core.engine import scrapyengine -from scrapy.http import Request -from scrapy.fetcher import get_or_create_spider +from scrapy.shell import Shell class Command(ScrapyCommand): @@ -31,100 +18,16 @@ class Command(ScrapyCommand): return "Interactive scraping console" def long_desc(self): - return "Interactive console for scraping the given url. For scraping local files you can use a URL like file://path/to/file.html" + return "Interactive console for scraping the given url. For scraping " \ + "local files you can use a URL like file://path/to/file.html" - def update_vars(self): - """ You can use this function to update the Scrapy objects that will be available in the shell""" + def update_vars(self, vars): + """You can use this function to update the Scrapy objects that will be + available in the shell + """ pass - def get_url(self, url): - url = url.strip() - if url: - u = urlparse.urlparse(url) - if not u.scheme: - path = os.path.abspath(url).replace(os.sep, '/') - url = 'file://' + urllib.pathname2url(path) - u = urlparse.urlparse(url) - - if u.scheme not in ('http', 'https', 'file'): - print "Unsupported scheme '%s' in URL: <%s>" % (u.scheme, url) - return - request = Request(url) - else: - request = self.user_ns['request'] - - spider = get_or_create_spider(url) - print "Fetching %s..." % request - response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, request, spider) - if response: - self.generate_vars(url, response, request) - return True - - def generate_vars(self, url=None, response=None, request=None): - itemcls = load_object(settings['DEFAULT_ITEM_CLASS']) - item = itemcls() - self.vars['item'] = item - if url: - self.vars['xxs'] = XmlXPathSelector(response) - self.vars['hxs'] = HtmlXPathSelector(response) - self.vars['url'] = url - self.vars['response'] = response - self.vars['request'] = request - self.vars['spider'] = spiders.fromurl(url) - self.update_vars() - self.user_ns.update(self.vars) - self.print_vars() - - def print_vars(self): - print '-' * 60 - print "Available Scrapy objects:" - for key, val in self.vars.iteritems(): - print " %s: %s" % (key, val) - print "Available commands:" - print " get [url]: Fetch a new URL or re-fetch current Request" - print " shelp: Prints this help." - print '-' * 60 - def run(self, args, opts): - self.vars = {} - self.user_ns = {} - url = None - if args: - url = args[0] - - print "Welcome to Scrapy shell!" - - def _console_thread(): - - def _get_magic(shell, arg): - self.get_url(arg) - def _help_magic(shell, _): - self.print_vars() - - if url: - result = self.get_url(url) - if not result: - self.generate_vars() - else: - self.generate_vars() - try: # use IPython if available - import IPython - shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns) - ip = shell.IP.getapi() - ip.expose_magic("get", _get_magic) - ip.expose_magic("shelp", _help_magic) - shell.mainloop() - reactor.callFromThread(scrapymanager.stop) - except ImportError: - import code - try: # readline module is only available on unix systems - import readline - except ImportError: - pass - else: - import rlcompleter - readline.parse_and_bind("tab:complete") - code.interact(local=self.vars) - - reactor.callInThread(_console_thread) - scrapymanager.start() + url = args[0] if args else None + shell = Shell(self.update_vars) + shell.start(url) diff --git a/scrapy/shell.py b/scrapy/shell.py new file mode 100644 index 000000000..0c579a40b --- /dev/null +++ b/scrapy/shell.py @@ -0,0 +1,127 @@ +""" +Scrapy Shell + +See documentation in docs/topics/shell.rst +""" + +import os +import urllib +import urlparse +import signal + +from twisted.internet import reactor, threads + +from scrapy.spider import spiders +from scrapy.selector import XmlXPathSelector, HtmlXPathSelector +from scrapy.utils.misc import load_object +from scrapy.utils.response import open_in_browser +from scrapy.conf import settings +from scrapy.core.manager import scrapymanager +from scrapy.core.engine import scrapyengine +from scrapy.http import Request +from scrapy.fetcher import get_or_create_spider + +def relevant_var(varname): + return varname not in ['shelp', 'fetch', 'view', '__builtins__', 'In', \ + 'Out', 'help'] and not varname.startswith('_') + +def parse_url(url): + """Parse url which can be a direct path to a direct file""" + url = url.strip() + if url: + u = urlparse.urlparse(url) + if not u.scheme: + path = os.path.abspath(url).replace(os.sep, '/') + url = 'file://' + urllib.pathname2url(path) + u = urlparse.urlparse(url) + return url + +class Shell(object): + + requires_project = False + + def __init__(self, update_vars): + self.vars = {} + self.update_vars = update_vars + self.item_class = load_object(settings['DEFAULT_ITEM_CLASS']) + + def fetch(self, request_or_url, print_help=False): + if isinstance(request_or_url, Request): + request = request_or_url + url = request.url + else: + url = parse_url(request_or_url) + request = Request(url) + spider = get_or_create_spider(url) + print "Fetching %s..." % request + response = threads.blockingCallFromThread(reactor, scrapyengine.schedule, \ + request, spider) + if response: + self.populate_vars(url, response, request) + if print_help: + self.print_help() + else: + print "Done - use shelp() to see available objects" + + def populate_vars(self, url=None, response=None, request=None): + item = self.item_class() + self.vars['item'] = item + if url: + self.vars['xxs'] = XmlXPathSelector(response) + self.vars['hxs'] = HtmlXPathSelector(response) + self.vars['url'] = url + self.vars['response'] = response + self.vars['request'] = request + self.vars['spider'] = spiders.fromurl(url) + + self.vars['fetch'] = self.fetch + self.vars['view'] = open_in_browser + self.vars['shelp'] = self.print_help + + self.update_vars(self.vars) + + def print_help(self): + print "Available objects" + print "=================" + print + for k, v in self.vars.iteritems(): + if relevant_var(k): + print " %-10s: %s" % (k, v) + print + print "Available shortcuts" + print "===================" + print + print " shelp() : Prints this help." + print " fetch(req_or_url) : Fetch a new request or URL and update objects" + print " view(response) : View response in a browser" + print + + def start(self, url): + # disable accidental Ctrl-C key press from shutting down the engine + signal.signal(signal.SIGINT, signal.SIG_IGN) + + reactor.callInThread(self._console_thread, url) + scrapymanager.start() + + def _console_thread(self, url=None): + self.populate_vars() + if url: + result = self.fetch(url, print_help=True) + else: + self.print_help() + try: # use IPython if available + import IPython + shell = IPython.Shell.IPShell(argv=[], user_ns=self.vars) + ip = shell.IP.getapi() + shell.mainloop() + except ImportError: + import code + try: # readline module is only available on unix systems + import readline + except ImportError: + pass + else: + import rlcompleter + readline.parse_and_bind("tab:complete") + code.interact(local=self.vars) + reactor.callFromThread(scrapymanager.stop)