diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index fc60488c5..cf40b01c2 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -17,8 +17,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" - def _extract_links(self, response_text, response_url, response_encoding): - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url + def _extract_links(self, response_text, response_url, response_encoding, base_url=None): + if base_url is None: + base_url = urljoin(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()