From be7821a48200e784e26f7b6160d44a854953b12e Mon Sep 17 00:00:00 2001 From: Mikhail Lyundin Date: Fri, 4 Sep 2015 20:35:27 +0300 Subject: [PATCH] Optimization - avoid temporary list objects, unnecessary function call --- scrapy/commands/check.py | 2 +- scrapy/commands/genspider.py | 4 ++-- scrapy/crawler.py | 2 +- scrapy/loader/__init__.py | 6 +++--- scrapy/shell.py | 2 +- scrapy/utils/defer.py | 2 +- scrapy/utils/request.py | 4 ++-- 7 files changed, 11 insertions(+), 11 deletions(-) diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 2917b8ba7..017595f04 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -62,7 +62,7 @@ class Command(ScrapyCommand): self.settings['SPIDER_CONTRACTS_BASE'], self.settings['SPIDER_CONTRACTS'], ) - conman = ContractsManager([load_object(c) for c in contracts]) + conman = ContractsManager(load_object(c) for c in contracts) runner = TextTestRunner(verbosity=2 if opts.verbose else 1) result = TextTestResult(runner.stream, runner.descriptions, runner.verbosity) diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py index 2c14b3c1f..f0d435379 100644 --- a/scrapy/commands/genspider.py +++ b/scrapy/commands/genspider.py @@ -88,8 +88,8 @@ class Command(ScrapyCommand): 'module': module, 'name': name, 'domain': domain, - 'classname': '%sSpider' % ''.join([s.capitalize() \ - for s in module.split('_')]) + 'classname': '%sSpider' % ''.join(s.capitalize() \ + for s in module.split('_')) } spiders_module = import_module(self.settings['NEWSPIDER_MODULE']) spiders_dir = abspath(dirname(spiders_module.__file__)) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 2f1a92d31..c7e3bb528 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -173,7 +173,7 @@ class CrawlerRunner(object): Returns a deferred that is fired when they all have ended. """ - return defer.DeferredList([c.stop() for c in list(self.crawlers)]) + return defer.DeferredList([c.stop() for c in self.crawlers]) @defer.inlineCallbacks def join(self): diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py index 84640f0b6..6c2ff968e 100644 --- a/scrapy/loader/__init__.py +++ b/scrapy/loader/__init__.py @@ -69,7 +69,7 @@ class ItemLoader(object): regex = kw.get('re', None) if regex: value = arg_to_iter(value) - value = flatten([extract_regex(regex, x) for x in value]) + value = flatten(extract_regex(regex, x) for x in value) for proc in processors: if value is None: @@ -149,7 +149,7 @@ class ItemLoader(object): def _get_xpathvalues(self, xpaths, **kw): self._check_selector_method() xpaths = arg_to_iter(xpaths) - return flatten([self.selector.xpath(xpath).extract() for xpath in xpaths]) + return flatten(self.selector.xpath(xpath).extract() for xpath in xpaths) def add_css(self, field_name, css, *processors, **kw): values = self._get_cssvalues(css, **kw) @@ -166,7 +166,7 @@ class ItemLoader(object): def _get_cssvalues(self, csss, **kw): self._check_selector_method() csss = arg_to_iter(csss) - return flatten([self.selector.css(css).extract() for css in csss]) + return flatten(self.selector.css(css).extract() for css in csss) XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader) diff --git a/scrapy/shell.py b/scrapy/shell.py index f008ce39a..099e1af0a 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -146,7 +146,7 @@ class Shell(object): "update local objects") b.append(" view(response) View response in a browser") - return "\n".join(["[s] %s" % l for l in b]) + return "\n".join("[s] %s" % l for l in b) def _is_relevant(self, value): return isinstance(value, self.relevant_classes) diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8f3824abf..bb4c74a6e 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -61,7 +61,7 @@ def parallel(iterable, count, callable, *args, **named): """ coop = task.Cooperator() work = (callable(elem, *args, **named) for elem in iterable) - return defer.DeferredList([coop.coiterate(work) for i in range(count)]) + return defer.DeferredList([coop.coiterate(work) for _ in range(count)]) def process_chain(callbacks, input, *a, **kw): """Return a Deferred built by chaining the given callbacks""" diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py index e361b7433..a98ca61ce 100644 --- a/scrapy/utils/request.py +++ b/scrapy/utils/request.py @@ -44,8 +44,8 @@ def request_fingerprint(request, include_headers=None): """ if include_headers: - include_headers = tuple([to_bytes(h.lower()) - for h in sorted(include_headers)]) + include_headers = tuple(to_bytes(h.lower()) + for h in sorted(include_headers)) cache = _fingerprint_cache.setdefault(request, {}) if include_headers not in cache: fp = hashlib.sha1()