diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst index 32669104c..6636c30cb 100644 --- a/docs/topics/commands.rst +++ b/docs/topics/commands.rst @@ -322,6 +322,14 @@ So this command can be used to "see" how your spider would fetch a certain page. If used outside a project, no particular per-spider behaviour would be applied and it will just use the default Scrapy downloader settings. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``--headers``: print the response's HTTP headers instead of the response's body + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them) + Usage examples:: $ scrapy fetch --nolog http://www.example.com/some/page.html @@ -368,11 +376,34 @@ given. Also supports UNIX-style local file paths, either relative with ``./`` or ``../`` prefixes or absolute file paths. See :ref:`topics-shell` for more info. +Supported options: + +* ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider + +* ``-c code``: evaluate the code in the shell, print the result and exit + +* ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them); + this only affects the URL you may pass as argument on the command line; + once you are inside the shell, ``fetch(url)`` will still follow HTTP redirects by default. + Usage example:: $ scrapy shell http://www.example.com/some/page.html [ ... scrapy shell starts ... ] + $ scrapy shell --nolog http://www.example.com/ -c '(response.status, response.url)' + (200, 'http://www.example.com/') + + # shell follows HTTP redirects by default + $ scrapy shell --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (200, 'http://example.com/') + + # you can disable this with --no-redirect + # (only for the URL passed as command line argument) + $ scrapy shell --no-redirect --nolog http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F -c '(response.status, response.url)' + (302, 'http://httpbin.org/redirect-to?url=http%3A%2F%2Fexample.com%2F') + + .. command:: parse parse diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst index da91108b2..ef6aeeed3 100644 --- a/docs/topics/shell.rst +++ b/docs/topics/shell.rst @@ -97,8 +97,12 @@ Available Shortcuts * ``shelp()`` - print a help with the list of available objects and shortcuts - * ``fetch(request_or_url)`` - fetch a new response from the given request or - URL and update all related objects accordingly. + * ``fetch(url[, redirect=True])`` - fetch a new response from the given + URL and update all related objects accordingly. You can optionaly ask for + HTTP 3xx redirections to not be followed by passing ``redirect=False`` + + * ``fetch(request)`` - fetch a new response from the given request and + update all related objects accordingly. * ``view(response)`` - open the given response in your local web browser, for inspection. This will add a `\ tag`_ to the response body in order @@ -157,36 +161,28 @@ list of available objects and useful shortcuts (you'll notice that these lines all start with the ``[s]`` prefix):: [s] Available Scrapy objects: - [s] crawler + [s] scrapy scrapy module (contains scrapy.Request, scrapy.Selector, etc) + [s] crawler [s] item {} [s] request - [s] response <200 http://scrapy.org> - [s] settings - [s] spider + [s] response <200 https://scrapy.org/> + [s] settings + [s] spider [s] Useful shortcuts: + [s] fetch(url[, redirect=True]) Fetch URL and update local objects (by default, redirects are followed) + [s] fetch(req) Fetch a scrapy.Request and update local objects [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects [s] view(response) View response in a browser >>> + After that, we can start playing with the objects:: >>> response.xpath('//title/text()').extract_first() 'Scrapy | A Fast and Powerful Scraping and Web Crawling Framework' >>> fetch("http://reddit.com") - [s] Available Scrapy objects: - [s] crawler - [s] item {} - [s] request - [s] response <200 https://www.reddit.com/> - [s] settings - [s] spider - [s] Useful shortcuts: - [s] shelp() Shell help (print this help) - [s] fetch(req_or_url) Fetch request (or URL) and update local objects - [s] view(response) View response in a browser >>> response.xpath('//title/text()').extract() ['reddit: the front page of the internet'] @@ -194,12 +190,36 @@ After that, we can start playing with the objects:: >>> request = request.replace(method="POST") >>> fetch(request) - [s] Available Scrapy objects: - [s] crawler - ... + >>> response.status + 404 + + >>> from pprint import pprint + + >>> pprint(response.headers) + {'Accept-Ranges': ['bytes'], + 'Cache-Control': ['max-age=0, must-revalidate'], + 'Content-Type': ['text/html; charset=UTF-8'], + 'Date': ['Thu, 08 Dec 2016 16:21:19 GMT'], + 'Server': ['snooserv'], + 'Set-Cookie': ['loid=KqNLou0V9SKMX4qb4n; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.445Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loid=vi0ZVe4NkxNWdlH7r7; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure', + 'loidcreated=2016-12-08T16%3A21%3A19.459Z; Domain=reddit.com; Max-Age=63071999; Path=/; expires=Sat, 08-Dec-2018 16:21:19 GMT; secure'], + 'Vary': ['accept-encoding'], + 'Via': ['1.1 varnish'], + 'X-Cache': ['MISS'], + 'X-Cache-Hits': ['0'], + 'X-Content-Type-Options': ['nosniff'], + 'X-Frame-Options': ['SAMEORIGIN'], + 'X-Moose': ['majestic'], + 'X-Served-By': ['cache-cdg8730-CDG'], + 'X-Timer': ['S1481214079.394283,VS0,VE159'], + 'X-Ua-Compatible': ['IE=edge'], + 'X-Xss-Protection': ['1; mode=block']} >>> + .. _topics-shell-inspect-response: Invoking the shell from spiders to inspect responses diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..7d4840529 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -5,6 +5,7 @@ from w3lib.url import is_url from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.exceptions import UsageError +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.spider import spidercls_for_request, DefaultSpider class Command(ScrapyCommand): @@ -27,6 +28,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +53,12 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + # by default, let the framework handle redirects, + # i.e. command handles all codes expect 3xx + if not opts.no_redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..40a58d94a 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \ + default=False, help="do not handle HTTP 3xx status codes and print response as-is") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, redirect=not opts.no_redirect) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 183ee1f70..6f94635a1 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -20,6 +20,7 @@ from scrapy.item import BaseItem from scrapy.settings import Settings from scrapy.spiders import Spider from scrapy.utils.console import start_python_console +from scrapy.utils.datatypes import SequenceExclude from scrapy.utils.misc import load_object from scrapy.utils.response import open_in_browser from scrapy.utils.conf import get_config @@ -40,11 +41,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, redirect=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, redirect=redirect) elif request: self.fetch(request, spider) elif response: @@ -98,14 +99,16 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, redirect=True, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if redirect: + request.meta['handle_httpstatus_list'] = SequenceExclude(range(300, 400)) + else: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread( @@ -144,10 +147,13 @@ class Shell(object): if self._is_relevant(v): b.append(" %-10s %s" % (k, v)) b.append("Useful shortcuts:") - b.append(" shelp() Shell help (print this help)") if self.inthread: - b.append(" fetch(req_or_url) Fetch request (or URL) and " - "update local objects") + b.append(" fetch(url[, redirect=True]) " + "Fetch URL and update local objects " + "(by default, redirects are followed)") + b.append(" fetch(req) " + "Fetch a scrapy.Request and update local objects ") + b.append(" shelp() Shell help (print this help)") b.append(" view(response) View response in a browser") return "\n".join("[s] %s" % l for l in b) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index d04b43176..e516185bd 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -304,3 +304,13 @@ class LocalCache(OrderedDict): while len(self) >= self.limit: self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) + + +class SequenceExclude(object): + """Object to test if an item is NOT within some sequence.""" + + def __init__(self, seq): + self.seq = seq + + def __contains__(self, item): + return item not in self.seq diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py index ad0375443..e50a989b3 100644 --- a/scrapy/utils/testsite.py +++ b/scrapy/utils/testsite.py @@ -20,12 +20,20 @@ class SiteTest(object): return urljoin(self.baseurl, path) +class NoMetaRefreshRedirect(util.Redirect): + def render(self, request): + content = util.Redirect.render(self, request) + return content.replace(b'http-equiv=\"refresh\"', + b'http-no-equiv=\"do-not-refresh-me\"') + + def test_site(): r = resource.Resource() r.putChild(b"text", static.Data(b"Works", "text/plain")) r.putChild(b"html", static.Data(b"

Works

World

", "text/html")) r.putChild(b"enc-gb18030", static.Data(b"

gb18030 encoding

", "text/html; charset=gb18030")) r.putChild(b"redirect", util.Redirect(b"/redirected")) + r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected")) r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain")) return server.Site(r) diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index 4843a9a2f..3fa3ed930 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -14,6 +14,18 @@ class FetchTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/text')]) self.assertEqual(out.strip(), b'Works') + @defer.inlineCallbacks + def test_redirect_default(self): + _, out, _ = yield self.execute([self.url('/redirect')]) + self.assertEqual(out.strip(), b'Redirected here') + + @defer.inlineCallbacks + def test_redirect_disabled(self): + _, out, err = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh')]) + err = err.strip() + self.assertIn(b'downloader/response_status_count/302', err, err) + self.assertNotIn(b'downloader/response_status_count/200', err, err) + @defer.inlineCallbacks def test_headers(self): _, out, _ = yield self.execute([self.url('/text'), '--headers']) diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py index 7bb7439d6..3e27d6abd 100644 --- a/tests/test_command_shell.py +++ b/tests/test_command_shell.py @@ -49,6 +49,35 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase): _, out, _ = yield self.execute([self.url('/redirect'), '-c', 'response.url']) assert out.strip().endswith(b'/redirected') + @defer.inlineCallbacks + def test_redirect_follow_302(self): + _, out, _ = yield self.execute([self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'200') + + @defer.inlineCallbacks + def test_redirect_not_follow_302(self): + _, out, _ = yield self.execute(['--no-redirect', self.url('/redirect-no-meta-refresh'), '-c', 'response.status']) + assert out.strip().endswith(b'302') + + @defer.inlineCallbacks + def test_fetch_redirect_follow_302(self): + """Test that calling `fetch(url)` follows HTTP redirects by default.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}')" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Redirecting (302)' in errout + assert b'Crawled (200)' in errout + + @defer.inlineCallbacks + def test_fetch_redirect_not_follow_302(self): + """Test that calling `fetch(url, redirect=False)` disables automatic redirects.""" + url = self.url('/redirect-no-meta-refresh') + code = "fetch('{0}', redirect=False)" + errcode, out, errout = yield self.execute(['-c', code.format(url)]) + self.assertEqual(errcode, 0, out) + assert b'Crawled (302)' in errout + @defer.inlineCallbacks def test_request_replace(self): url = self.url('/text') diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index b31d2179c..80f797227 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -1,7 +1,7 @@ import copy import unittest -from scrapy.utils.datatypes import CaselessDict +from scrapy.utils.datatypes import CaselessDict, SequenceExclude __doctests__ = ['scrapy.utils.datatypes'] @@ -128,6 +128,67 @@ class CaselessDictTest(unittest.TestCase): assert isinstance(h2, CaselessDict) +class SequenceExcludeTest(unittest.TestCase): + + def test_list(self): + seq = [1, 2, 3] + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn(4, d) + self.assertNotIn(2, d) + + def test_range(self): + seq = range(10, 20) + d = SequenceExclude(seq) + self.assertIn(5, d) + self.assertIn(20, d) + self.assertNotIn(15, d) + + def test_six_range(self): + import six.moves + seq = six.moves.range(10**3, 10**6) + d = SequenceExclude(seq) + self.assertIn(10**2, d) + self.assertIn(10**7, d) + self.assertNotIn(10**4, d) + + def test_range_step(self): + seq = range(10, 20, 3) + d = SequenceExclude(seq) + are_not_in = [v for v in range(10, 20, 3) if v in d] + self.assertEquals([], are_not_in) + + are_not_in = [v for v in range(10, 20) if v in d] + self.assertEquals([11, 12, 14, 15, 17, 18], are_not_in) + + def test_string_seq(self): + seq = "cde" + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_stringset_seq(self): + seq = set("cde") + d = SequenceExclude(seq) + chars = "".join(v for v in "abcdefg" if v in d) + self.assertEquals("abfg", chars) + + def test_set(self): + """Anything that is not in the supplied sequence will evaluate as 'in' the container.""" + seq = set([-3, "test", 1.1]) + d = SequenceExclude(seq) + self.assertIn(0, d) + self.assertIn("foo", d) + self.assertIn(3.14, d) + self.assertIn(set("bar"), d) + + # supplied sequence is a set, so checking for list (non)inclusion fails + self.assertRaises(TypeError, (0, 1, 2) in d) + self.assertRaises(TypeError, d.__contains__, ['a', 'b', 'c']) + + for v in [-3, "test", 1.1]: + self.assertNotIn(v, d) + if __name__ == "__main__": unittest.main()