RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added stripping of urls in LinkExtractor, and removed unnecesary check 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40459
This commit is contained in:
elpolilla 2008-12-02 13:26:04 +00:00
parent c22223beb7
commit a4ba7b9d9a
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/trunk/scrapy/link/extractors.py
View File

@ -55,7 +55,7 @@ class RegexLinkExtractor(LinkExtractor):
response = new_response_from_xpaths(response, self.restrict_xpaths)
links = LinkExtractor.extract_urls(self, response, unique)
links = [link for link in links if _is_valid_url(link.url)]
links = [link.strip() for link in links if _is_valid_url(link.url)]
if self.allow_res:
links = [link for link in links if _matches(link.url, self.allow_res)]
@ -80,4 +80,4 @@ class RegexLinkExtractor(LinkExtractor):
allowed = [regex.search(url) for regex in self.allow_res] if self.allow_res else [True]
denied = [regex.search(url) for regex in self.deny_res] if self.deny_res else []
return True if any(allowed) and not any(denied) else False
return any(allowed) and not any(denied)