From 2c14692e603873cf5b5783ad62ebe47474d2d7a8 Mon Sep 17 00:00:00 2001
From: s-sanjay <7111850+s-sanjay@users.noreply.github.com>
Date: Fri, 27 Sep 2019 00:56:43 -0700
Subject: [PATCH] remove .keys() to avoid creating a tmp list/keyview obj
 (#4031)

Also add --verbose and --nolinks for code coverage
---
 scrapy/commands/parse.py     | 15 ++++++++-------
 scrapy/commands/runspider.py |  5 ++---
 tests/test_command_parse.py  |  6 ++++++
 3 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index d4f2234b0..ef8acd29c 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -60,10 +60,12 @@ class Command(ScrapyCommand):
 
     @property
     def max_level(self):
-        levels = list(self.items.keys()) + list(self.requests.keys())
-        if not levels:
-            return 0
-        return max(levels)
+        max_items, max_requests = 0, 0
+        if self.items:
+            max_items = max(self.items)
+        if self.requests:
+            max_requests = max(self.requests)
+        return max(max_items, max_requests)
 
     def add_items(self, lvl, new_items):
         old_items = self.items.get(lvl, [])
@@ -84,9 +86,8 @@ class Command(ScrapyCommand):
 
     def print_requests(self, lvl=None, colour=True):
         if lvl is None:
-            levels = list(self.requests.keys())
-            if levels:
-                requests = self.requests[max(levels)]
+            if self.requests:
+                requests = self.requests[max(self.requests)]
             else:
                 requests = []
         else:
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 376d3c84e..57d8471ca 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -60,14 +60,13 @@ class Command(ScrapyCommand):
             else:
                 self.settings.set('FEED_URI', opts.output, priority='cmdline')
             feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
-            valid_output_formats = feed_exporters.keys()
             if not opts.output_format:
                 opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
-            if opts.output_format not in valid_output_formats:
+            if opts.output_format not in feed_exporters:
                 raise UsageError("Unrecognized output format '%s', set one"
                                  " using the '-t' switch or as a file extension"
                                  " from the supported list %s" % (opts.output_format,
-                                                                  tuple(valid_output_formats)))
+                                                                  tuple(feed_exporters)))
             self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
 
     def run(self, args, opts):
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 98e415ad3..62d5d76b4 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -108,6 +108,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-a', 'test_arg=1',
                                            '-c', 'parse',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -117,12 +118,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--meta', raw_json_string,
                                            '-c', 'parse_request_with_meta',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '-m', raw_json_string,
                                            '-c', 'parse_request_with_meta',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -132,6 +135,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--cbkwargs', raw_json_string,
                                            '-c', 'parse_request_with_cb_kwargs',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -139,6 +143,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
     def test_request_without_meta(self):
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                           '-c', 'parse_request_without_meta',
+                                          '--nolinks',
                                            self.url('/html')])
         self.assertIn("DEBUG: It Works!", _textmode(stderr))
 
@@ -148,6 +153,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
         _, _, stderr = yield self.execute(['--spider', self.spider_name,
                                            '--pipelines',
                                            '-c', 'parse',
+                                           '--verbose',
                                            self.url('/html')])
         self.assertIn("INFO: It Works!", _textmode(stderr))