RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

_extract_links requires extra parameter base_url in order to avoid 

exception when called from superclass method
This commit is contained in:
Martin Olveyra 2011-09-14 10:58:09 -03:00
parent ab1c9cfc56
commit 509b05db57
1 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/contrib/linkextractors/regex.py
View File

@ -17,8 +17,9 @@ def clean_link(link_text):
class RegexLinkExtractor(SgmlLinkExtractor):
"""High performant link extractor"""
def _extract_links(self, response_text, response_url, response_encoding):
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
def _extract_links(self, response_text, response_url, response_encoding, base_url=None):
if base_url is None:
base_url = urljoin(response_url, self.base_url) if self.base_url else response_url
clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()