From f73eb715ac4374b1eafb1da3c1311c5a7c153b5e Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Tue, 7 Feb 2017 02:16:06 +0500 Subject: [PATCH] =?UTF-8?q?LinkExtractor:=20don=E2=80=99t=20check=20all=20?= =?UTF-8?q?regexes=20if=20one=20of=20them=20matches?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/linkextractors/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py index f51934b00..e5d21e174 100644 --- a/scrapy/linkextractors/__init__.py +++ b/scrapy/linkextractors/__init__.py @@ -40,7 +40,7 @@ IGNORED_EXTENSIONS = [ _re_type = type(re.compile("", 0)) -_matches = lambda url, regexs: any((r.search(url) for r in regexs)) +_matches = lambda url, regexs: any(r.search(url) for r in regexs) _is_valid_url = lambda url: url.split('://', 1)[0] in {'http', 'https', 'file'} @@ -93,8 +93,8 @@ class FilteringLinkExtractor(object): if self.deny_domains and url_is_from_any_domain(url, self.deny_domains): return False - allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] - denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] + allowed = (regex.search(url) for regex in self.allow_res) if self.allow_res else [True] + denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else [] return any(allowed) and not any(denied) def _process_links(self, links):