RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

remove .keys() to avoid creating a tmp list/keyview obj (#4031) 

Also add --verbose and --nolinks for code coverage
This commit is contained in:
s-sanjay 2019-09-27 00:56:43 -07:00 committed by Adrián Chaves
parent 31c631f9ee
commit 2c14692e60
3 changed files with 16 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
scrapy/commands/parse.py
View File

@ -60,10 +60,12 @@ class Command(ScrapyCommand):
@property
def max_level(self):
levels = list(self.items.keys()) + list(self.requests.keys())
if not levels:
return 0
return max(levels)
max_items, max_requests = 0, 0
if self.items:
max_items = max(self.items)
if self.requests:
max_requests = max(self.requests)
return max(max_items, max_requests)
def add_items(self, lvl, new_items):
old_items = self.items.get(lvl, [])
@ -84,9 +86,8 @@ class Command(ScrapyCommand):
def print_requests(self, lvl=None, colour=True):
if lvl is None:
levels = list(self.requests.keys())
if levels:
requests = self.requests[max(levels)]
if self.requests:
requests = self.requests[max(self.requests)]
else:
requests = []
else:

5
scrapy/commands/runspider.py
View File

@ -60,14 +60,13 @@ class Command(ScrapyCommand):
else:
self.settings.set('FEED_URI', opts.output, priority='cmdline')
feed_exporters = without_none_values(self.settings.getwithbase('FEED_EXPORTERS'))
valid_output_formats = feed_exporters.keys()
if not opts.output_format:
opts.output_format = os.path.splitext(opts.output)[1].replace(".", "")
if opts.output_format not in valid_output_formats:
if opts.output_format not in feed_exporters:
raise UsageError("Unrecognized output format '%s', set one"
" using the '-t' switch or as a file extension"
" from the supported list %s" % (opts.output_format,
tuple(valid_output_formats)))
tuple(feed_exporters)))
self.settings.set('FEED_FORMAT', opts.output_format, priority='cmdline')
def run(self, args, opts):

6
tests/test_command_parse.py
View File

@ -108,6 +108,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-a', 'test_arg=1',
'-c', 'parse',
'--verbose',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@ -117,12 +118,14 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'--meta', raw_json_string,
'-c', 'parse_request_with_meta',
'--verbose',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-m', raw_json_string,
'-c', 'parse_request_with_meta',
'--verbose',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@ -132,6 +135,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'--cbkwargs', raw_json_string,
'-c', 'parse_request_with_cb_kwargs',
'--verbose',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@ -139,6 +143,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
def test_request_without_meta(self):
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'-c', 'parse_request_without_meta',
'--nolinks',
self.url('/html')])
self.assertIn("DEBUG: It Works!", _textmode(stderr))
@ -148,6 +153,7 @@ ITEM_PIPELINES = {'%s.pipelines.MyPipeline': 1}
_, _, stderr = yield self.execute(['--spider', self.spider_name,
'--pipelines',
'-c', 'parse',
'--verbose',
self.url('/html')])
self.assertIn("INFO: It Works!", _textmode(stderr))