Handle redirects transparently by default in shell and fetch

Adds --no-status-aware command line option to have previous behaviour

scrapy/commands/fetch.py

@@ -27,6 +27,8 @@ class Command(ScrapyCommand):
             help="use this spider")
         parser.add_option("--headers", dest="headers", action="store_true", \
             help="print response HTTP headers instead of body")
+        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+            default=False, help="do not handle status codes like redirects and print response as-is")
 
     def _print_headers(self, headers, prefix):
         for key, values in headers.items():
@@ -50,7 +52,8 @@ class Command(ScrapyCommand):
             raise UsageError()
         cb = lambda x: self._print_response(x, opts)
         request = Request(args[0], callback=cb, dont_filter=True)
-        request.meta['handle_httpstatus_all'] = True
+        if opts.no_status_aware:
+            request.meta['handle_httpstatus_all'] = True
 
         spidercls = DefaultSpider
         spider_loader = self.crawler_process.spider_loader

scrapy/commands/shell.py

@@ -36,6 +36,8 @@ class Command(ScrapyCommand):
             help="evaluate the code in the shell, print the result and exit")
         parser.add_option("--spider", dest="spider",
             help="use this spider")
+        parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \
+            default=False, help="do not transparently handle status codes like redirects")
 
     def update_vars(self, vars):
         """You can use this function to update the Scrapy objects that will be
@@ -68,7 +70,7 @@ class Command(ScrapyCommand):
         self._start_crawler_thread()
 
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
-        shell.start(url=url)
+        shell.start(url=url, handle_statuses=opts.no_status_aware)
 
     def _start_crawler_thread(self):
         t = Thread(target=self.crawler_process.start,

scrapy/shell.py

@@ -40,11 +40,11 @@ class Shell(object):
         self.code = code
         self.vars = {}
 
-    def start(self, url=None, request=None, response=None, spider=None):
+    def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True):
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
-            self.fetch(url, spider)
+            self.fetch(url, spider, handle_statuses=handle_statuses)
         elif request:
             self.fetch(request, spider)
         elif response:
@@ -98,14 +98,14 @@ class Shell(object):
         self.spider = spider
         return spider
 
-    def fetch(self, request_or_url, spider=None):
+    def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs):
         if isinstance(request_or_url, Request):
             request = request_or_url
-            url = request.url
         else:
             url = any_to_uri(request_or_url)
-            request = Request(url, dont_filter=True)
-            request.meta['handle_httpstatus_all'] = True
+            request = Request(url, dont_filter=True, **kwargs)
+            if handle_statuses:
+                request.meta['handle_httpstatus_all'] = True
         response = None
         try:
             response, spider = threads.blockingCallFromThread(