added inspect_response() function for inspecting responses from spiders

docs/topics/shell.rst

@@ -53,8 +53,12 @@ Built-in Shortcuts
    URL and update all related objects accordingly.
 
  * ``view(response)`` - open the given response in your local web browser, for
-   inspection. Note that this will generate a temporary file which won't be
-   removed automatically.
+   inspection. This will add a `\<base\> tag`_ to the response body in order
+   for external links (such as images and style sheets) to display properly.
+   Note, however,that this will create a temporary file in your computer,
+   which won't be removed automatically.
+
+.. _<base> tag: http://www.w3schools.com/TAGS/tag_base.asp
 
 Built-in Objects
 ----------------
@@ -77,7 +81,7 @@ Those objects are:
    fetch a new request (without leaving the shell) using the ``fetch``
    shortcut.
 
- * ``response`` - a :class:`~scrapy.http.Response` object contaning the last
+ * ``response`` - a :class:`~scrapy.http.Response` object containing the last
    fetched page
 
  * ``hxs`` - a :class:`~scrapy.selector.HtmlXPathSelector` object constructed
@@ -146,3 +150,66 @@ After that, we can stary playing with the objects::
     2009-04-03 00:57:39-0300 [scrapybot] ERROR: Downloading <http://slashdot.org> from <None>: 405 Method Not Allowed
     >>> 
 
+
+Invoking the shell from spiders to inspect responses
+====================================================
+
+Sometimes you want to inspect the responses that are being processed in a
+certain point of your spider, if only to check that response you expect is
+getting there.
+
+This can be achieved by using the ``scrapy.shell.inspect_response`` function.
+
+Here's an example of how you would call it from your spider::
+
+    class MySpider(BaseSpider):
+        domain_name = 'example.com'
+
+        def parse(self, response):
+            if response.url == 'http://www.example.com/products.php':
+                from scrapy.shell import inspect_response
+                inspect_response(response)
+
+            # ... your parsing code ..
+
+When you the spider you will get something similar to this::
+
+    2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled <http://www.example.com/> (referer: <None>)
+    2009-08-27 19:15:26-0300 [example.com] DEBUG: Crawled <http://www.example.com/products.php> (referer: <http://www.example.com/>)
+
+    Scrapy Shell
+    ============
+
+    Inspecting: <http://www.example.com/products.php
+    Use shelp() to see available objects
+
+    Python 2.6.2 (release26-maint, Apr 19 2009, 01:58:18) 
+    [GCC 4.3.3] on linux2
+    Type "help", "copyright", "credits" or "license" for more information.
+    (InteractiveConsole)
+    >>> response.url
+    'http://www.example.com/products.php'
+
+Then, you can check if the extraction code is working::
+
+    >>> hxs.select('//h1')
+    []
+
+Nope, it doesn't. So you can open the response in your web browser and see if
+it's the response you were expecting::
+
+    >>> view(response)
+    >>>
+
+Finally you hit Ctrl-D (or Ctrl-Z in Windows) to exit the shell and resume the
+crawling::
+
+    >>> ^D
+    2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled <http://www.example.com/product.php?id=1> (referer: <None>)
+    2009-08-27 19:15:25-0300 [example.com] DEBUG: Crawled <http://www.example.com/product.php?id=2> (referer: <None>)
+    # ...
+
+Note that you can't use the ``fetch`` shortcut here since the Scrapy engine is
+blocked by the shell. However, after you leave the shell, the spider will
+continue crawling where it stopped, as shown above.
+

scrapy/log.py

@@ -34,6 +34,8 @@ log_level = DEBUG
 
 started = False
 
+_prev_descriptors = (sys.stdout, sys.stderr)
+
 def start(logfile=None, loglevel=None, logstdout=None):
     """Initialize and start logging facility"""
     global log_level, started
@@ -53,6 +55,13 @@ def start(logfile=None, loglevel=None, logstdout=None):
         file = open(logfile, 'a') if logfile else sys.stderr
         log.startLogging(file, setStdout=logstdout)
 
+def _switch_descriptors():
+    global _prev_descriptors
+
+    cur = (sys.stdout, sys.stderr)
+    sys.stdout, sys.stderr = _prev_descriptors
+    _prev_descriptors = cur
+
 def msg(message, level=INFO, component=BOT_NAME, domain=None):
     """Log message according to the level"""
     dispatcher.send(signal=logmessage_received, message=message, level=level, \

scrapy/shell.py

@@ -20,6 +20,7 @@ from scrapy.core.manager import scrapymanager
 from scrapy.core.engine import scrapyengine
 from scrapy.http import Request
 from scrapy.fetcher import get_or_create_spider
+from scrapy import log
 
 def relevant_var(varname):
     return varname not in ['shelp', 'fetch', 'view', '__builtins__', 'In', \
@@ -40,10 +41,11 @@ class Shell(object):
 
     requires_project = False
 
-    def __init__(self, update_vars):
+    def __init__(self, update_vars=None, nofetch=False):
         self.vars = {}
         self.update_vars = update_vars
         self.item_class = load_object(settings['DEFAULT_ITEM_CLASS'])
+        self.nofetch = nofetch
 
     def fetch(self, request_or_url, print_help=False):
         if isinstance(request_or_url, Request):
@@ -74,11 +76,13 @@ class Shell(object):
             self.vars['request'] = request
             self.vars['spider'] = spiders.fromurl(url)
 
-            self.vars['fetch'] = self.fetch
+            if not self.nofetch:
+                self.vars['fetch'] = self.fetch
             self.vars['view'] = open_in_browser
             self.vars['shelp'] = self.print_help
 
-        self.update_vars(self.vars)
+        if self.update_vars:
+            self.update_vars(self.vars)
 
     def print_help(self):
         print "Available objects"
@@ -92,7 +96,8 @@ class Shell(object):
         print "==================="
         print
         print "  shelp()           : Prints this help."
-        print "  fetch(req_or_url) : Fetch a new request or URL and update objects"
+        if not self.nofetch:
+            print "  fetch(req_or_url) : Fetch a new request or URL and update objects"
         print "  view(response)    : View response in a browser"
         print
 
@@ -103,12 +108,20 @@ class Shell(object):
         reactor.callInThread(self._console_thread, url)
         scrapymanager.start()
 
-    def _console_thread(self, url=None):
-        self.populate_vars()
-        if url:
-            result = self.fetch(url, print_help=True)
-        else:
-            self.print_help()
+    def inspect_response(self, response):
+        print
+        print "Scrapy Shell"
+        print "============"
+        print
+        print "Inspecting: %s" % response
+        print "Use shelp() to see available objects"
+        print
+        request = response.request
+        url = request.url
+        self.populate_vars(url, response, request)
+        self._run_console()
+
+    def _run_console(self):
         try: # use IPython if available
             import IPython
             shell = IPython.Shell.IPShell(argv=[], user_ns=self.vars)
@@ -124,4 +137,19 @@ class Shell(object):
                 import rlcompleter
                 readline.parse_and_bind("tab:complete")
             code.interact(local=self.vars)
+
+    def _console_thread(self, url=None):
+        self.populate_vars()
+        if url:
+            result = self.fetch(url, print_help=True)
+        else:
+            self.print_help()
+        self._run_console()
         reactor.callFromThread(scrapymanager.stop)
+
+def inspect_response(response):
+    """Open a shell to inspect the given response"""
+    shell = Shell(nofetch=True)
+    log._switch_descriptors()
+    shell.inspect_response(response)
+    log._switch_descriptors()