From a4ba7b9d9af84c727e9a0aae15cb7603151d63e3 Mon Sep 17 00:00:00 2001 From: elpolilla Date: Tue, 2 Dec 2008 13:26:04 +0000 Subject: [PATCH] Added stripping of urls in LinkExtractor, and removed unnecesary check --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40459 --- scrapy/trunk/scrapy/link/extractors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/trunk/scrapy/link/extractors.py b/scrapy/trunk/scrapy/link/extractors.py index 49adac8bd..70cc7e772 100644 --- a/scrapy/trunk/scrapy/link/extractors.py +++ b/scrapy/trunk/scrapy/link/extractors.py @@ -55,7 +55,7 @@ class RegexLinkExtractor(LinkExtractor): response = new_response_from_xpaths(response, self.restrict_xpaths) links = LinkExtractor.extract_urls(self, response, unique) - links = [link for link in links if _is_valid_url(link.url)] + links = [link.strip() for link in links if _is_valid_url(link.url)] if self.allow_res: links = [link for link in links if _matches(link.url, self.allow_res)] @@ -80,4 +80,4 @@ class RegexLinkExtractor(LinkExtractor): allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True] denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else [] - return True if any(allowed) and not any(denied) else False + return any(allowed) and not any(denied)