RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #38 from kalessin/master 

_extract_links require extra parameter base_url
This commit is contained in:
Pablo Hoffman 2011-09-14 07:58:43 -07:00
parent ab1c9cfc56 509b05db57
commit 77ffaa50e5
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/contrib/linkextractors/regex.py
View File

@ -17,8 +17,9 @@ def clean_link(link_text):
class RegexLinkExtractor(SgmlLinkExtractor):
"""High performant link extractor"""
def _extract_links(self, response_text, response_url, response_encoding):
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()