diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index f09a873c1..a157b19f8 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -27,6 +27,8 @@ class Command(ScrapyCommand): help="use this spider") parser.add_option("--headers", dest="headers", action="store_true", \ help="print response HTTP headers instead of body") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not handle status codes like redirects and print response as-is") def _print_headers(self, headers, prefix): for key, values in headers.items(): @@ -50,7 +52,8 @@ class Command(ScrapyCommand): raise UsageError() cb = lambda x: self._print_response(x, opts) request = Request(args[0], callback=cb, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + if opts.no_status_aware: + request.meta['handle_httpstatus_all'] = True spidercls = DefaultSpider spider_loader = self.crawler_process.spider_loader diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 7be7f7256..bc0203d89 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -36,6 +36,8 @@ class Command(ScrapyCommand): help="evaluate the code in the shell, print the result and exit") parser.add_option("--spider", dest="spider", help="use this spider") + parser.add_option("--no-status-aware", dest="no_status_aware", action="store_true", \ + default=False, help="do not transparently handle status codes like redirects") def update_vars(self, vars): """You can use this function to update the Scrapy objects that will be @@ -68,7 +70,7 @@ class Command(ScrapyCommand): self._start_crawler_thread() shell = Shell(crawler, update_vars=self.update_vars, code=opts.code) - shell.start(url=url) + shell.start(url=url, handle_statuses=opts.no_status_aware) def _start_crawler_thread(self): t = Thread(target=self.crawler_process.start, diff --git a/scrapy/shell.py b/scrapy/shell.py index 183ee1f70..966003f17 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -40,11 +40,11 @@ class Shell(object): self.code = code self.vars = {} - def start(self, url=None, request=None, response=None, spider=None): + def start(self, url=None, request=None, response=None, spider=None, handle_statuses=True): # disable accidental Ctrl-C key press from shutting down the engine signal.signal(signal.SIGINT, signal.SIG_IGN) if url: - self.fetch(url, spider) + self.fetch(url, spider, handle_statuses=handle_statuses) elif request: self.fetch(request, spider) elif response: @@ -98,14 +98,14 @@ class Shell(object): self.spider = spider return spider - def fetch(self, request_or_url, spider=None): + def fetch(self, request_or_url, spider=None, handle_statuses=False, **kwargs): if isinstance(request_or_url, Request): request = request_or_url - url = request.url else: url = any_to_uri(request_or_url) - request = Request(url, dont_filter=True) - request.meta['handle_httpstatus_all'] = True + request = Request(url, dont_filter=True, **kwargs) + if handle_statuses: + request.meta['handle_httpstatus_all'] = True response = None try: response, spider = threads.blockingCallFromThread(