From 509b05db571aac659c51ec36b75f83166a2eda49 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Wed, 14 Sep 2011 10:58:09 -0300 Subject: [PATCH] _extract_links requires extra parameter base_url in order to avoid exception when called from superclass method --- scrapy/contrib/linkextractors/regex.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index fc60488c5..cf40b01c2 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -17,8 +17,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" - def _extract_links(self, response_text, response_url, response_encoding): - base_url = urljoin(response_url, self.base_url) if self.base_url else response_url + def _extract_links(self, response_text, response_url, response_encoding, base_url=None): + if base_url is None: + base_url = urljoin(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()