diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..3a26b19ae 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,31 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index 322c3ddfa..6eb81a71f 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() u'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() [u'reddit: the front page of the internet'] @@ -194,12 +190,36 @@ After that, we can start playing with the objects:: >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses diff --git a/scrapy/shell.py b/scrapy/shell.py index 6c78722be..babc267c7 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -148,10 +148,13 @@ class Shell(object): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b)