From 0fb3ee16d8c413c5084d58bbd5bbef3441cb4a53 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Fri, 3 Apr 2009 04:13:21 +0000 Subject: [PATCH] more improvements to scrapy shell: added Request object, and support for modifying it and re-fetching it by issuing an empty 'get' command --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%401037 --- scrapy/trunk/docs/topics/shell.rst | 48 ++++++++++++---- scrapy/trunk/scrapy/command/commands/shell.py | 57 +++++++++++-------- 2 files changed, 72 insertions(+), 33 deletions(-) diff --git a/scrapy/trunk/docs/topics/shell.rst b/scrapy/trunk/docs/topics/shell.rst index 19ec83259..ed332673f 100644 --- a/scrapy/trunk/docs/topics/shell.rst +++ b/scrapy/trunk/docs/topics/shell.rst @@ -50,8 +50,11 @@ Custom Shell Commands * ``%shelp`` - show the status of your Scrapy objects and get a list of all (Scrapy related) available objects. - * ``%get ``- download a new response from the given URL and update all - Scrapy objects accordingly + * ``%get [url]``- fetch a new response from the given URL and update all + Scrapy objects accordingly. If the url is omitted, the last request is + re-fetched. Since Requests are mutable objects, you can modify the Request + "in place" and issue ``get`` to fetch it again with the modifications you've + made. These commands can be typed without the leading percent sign if you have `automagic commands`_ enabled in IPython. @@ -73,7 +76,12 @@ Those objects are: or a :class:`~scrapy.spider.BaseSpider` object if there is not spider configured to process that URL - * ``response`` - a Response object of the downloaded page + * ``request`` - a :class:`~scrapy.http.Request` object of the last fetched + page. You can modify this Request "in place" and re-fetch it using the + command ``get`` without no argument. + + * ``response`` - a :class:`~scrapy.http.Response` object of the last fetched + page * ``hxs`` - a :class:`~scrapy.xpath.HtmlXPathSelector` object for the Response of the downloaded page @@ -89,23 +97,29 @@ Example of shell session ======================== Here's an example of a typical shell session where we start by scraping the -http://scrapy.org page, and then the http://slashdot.org page. Note, however, -that the data extracted here may not be the same when you try, as those pages -are not static and could have changed by the time you test this. The purpose of -this example is just to get you familiarized with how the Scrapy shell works. +http://scrapy.org page, and then proceed to scrape the http://slashdot.org +page. Finally, we modify the (slashdot) request method to POST and re-fetch it +getting a HTTP 405 (method not allowed) error. We end the session by typing +Ctrl-D (in the ``In [6]`` prompt). + +Keep in mind that the data extracted here may not be the same when you try it, +as those pages are not static and could have changed by the time you test this. +The only purpose of this example is to get you familiarized with how the Scrapy +shell works. :: scrapy-ctl.py shell http://scrapy.org 2009-04-02 16:56:22-0300 [-] Log opened. - Starting Scrapy 0.7.0 shell... - Downloading URL... Done. + Welcome to Scrapy shell! + Fetching ... ------------------------------------------------------------ Available Scrapy variables: xxs: url: http://scrapy.org + request: spider: hxs: item: @@ -127,11 +141,12 @@ this example is just to get you familiarized with how the Scrapy shell works. Out[1]: u'Welcome to Scrapy' In [2]: get http://slashdot.org - Downloading URL... Done. + Fetching ... ------------------------------------------------------------ Available Scrapy variables: xxs: url: http://slashdot.org + request: spider: hxs: item: @@ -144,3 +159,16 @@ this example is just to get you familiarized with how the Scrapy shell works. In [3]: hxs.x("//h2/text()").extract() Out[3]: [u'News for nerds, stuff that matters'] + In [3]: hxs.x("//h2/text()").extract() + Out[3]: [u'News for nerds, stuff that matters'] + + In [4]: request.method = "POST" + + In [5]: get + Fetching ... + 2009-04-03 00:57:39-0300 [decobot/None] ERROR: Downloading from : 405 Method Not Allowed + + In [6]: + 2009-04-03 01:07:12-0300 [-] Main loop terminated. + + diff --git a/scrapy/trunk/scrapy/command/commands/shell.py b/scrapy/trunk/scrapy/command/commands/shell.py index c1c9ae53c..8ef9eb94e 100644 --- a/scrapy/trunk/scrapy/command/commands/shell.py +++ b/scrapy/trunk/scrapy/command/commands/shell.py @@ -1,3 +1,9 @@ +""" +Scrapy Shell + +See documentation in docs/topics/shell.rst +""" + import os import urllib import urlparse @@ -28,32 +34,36 @@ class Command(ScrapyCommand): """ You can use this function to update the Scrapy objects that will be available in the shell""" pass + def _get_response(self, response): + self.response = response + return [] + def get_url(self, url): - u = urlparse.urlparse(url) - if not u.scheme: - path = os.path.abspath(url).replace(os.sep, '/') - url = 'file://' + urllib.pathname2url(path) + self.response = None + + url = url.strip() + if url: u = urlparse.urlparse(url) + if not u.scheme: + path = os.path.abspath(url).replace(os.sep, '/') + url = 'file://' + urllib.pathname2url(path) + u = urlparse.urlparse(url) - if u.scheme not in ('http', 'https', 'file'): - print "Unsupported scheme '%s' in URL: <%s>" % (u.scheme, url) - return + if u.scheme not in ('http', 'https', 'file'): + print "Unsupported scheme '%s' in URL: <%s>" % (u.scheme, url) + return + request = Request(url, callback=self._get_response) + else: + request = self.user_ns['request'].replace(callback=self._get_response) - self.result = None - def _get_response(response): - self.result = response - return [] - - print "Downloading URL..." - r = Request(url, callback=_get_response) spider = get_or_create_spider(url) - threads.blockingCallFromThread(reactor, scrapyengine.crawl, r, spider) - if self.result: - self.result.request = r - self.generate_vars(url, self.result) + print "Fetching %s..." % request + threads.blockingCallFromThread(reactor, scrapyengine.crawl, request, spider) + if self.response: + self.generate_vars(url, self.response, request) return True - def generate_vars(self, url, response): + def generate_vars(self, url=None, response=None, request=None): itemcls = load_object(settings['DEFAULT_ITEM_CLASS']) item = itemcls() self.vars['item'] = item @@ -62,6 +72,7 @@ class Command(ScrapyCommand): self.vars['hxs'] = HtmlXPathSelector(response) self.vars['url'] = url self.vars['response'] = response + self.vars['request'] = request self.vars['spider'] = spiders.fromurl(url) self.update_vars() self.user_ns.update(self.vars) @@ -73,7 +84,7 @@ class Command(ScrapyCommand): for key, val in self.vars.iteritems(): print " %s: %s" % (key, val) print "Available commands:" - print " get : Fetches a new page and updates all Scrapy objects." + print " get [url]: Fetch a new URL or re-fetch current Request" print " shelp: Prints this help." print '-' * 60 @@ -89,16 +100,16 @@ class Command(ScrapyCommand): def _console_thread(): def _get_magic(shell, arg): - self.get_url(arg.strip()) + self.get_url(arg) def _help_magic(shell, _): self.print_vars() if url: result = self.get_url(url) if not result: - self.generate_vars(None, None) + self.generate_vars() else: - self.generate_vars(None, None) + self.generate_vars() try: # use IPython if available import IPython shell = IPython.Shell.IPShell(argv=[], user_ns=self.user_ns)