diff --git a/scrapy/contrib/linkextractors/htmlparser.py b/scrapy/contrib/linkextractors/htmlparser.py index 2714fb562..fb3fd661b 100644 --- a/scrapy/contrib/linkextractors/htmlparser.py +++ b/scrapy/contrib/linkextractors/htmlparser.py @@ -26,7 +26,7 @@ class HtmlParserLinkExtractor(HTMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/image.py b/scrapy/contrib/linkextractors/image.py index 79048a496..88dd78577 100644 --- a/scrapy/contrib/linkextractors/image.py +++ b/scrapy/contrib/linkextractors/image.py @@ -51,8 +51,7 @@ class HTMLImageLinkExtractor(object): def extract_links(self, response): xs = HtmlXPathSelector(response) base_url = xs.select('//base/@href').extract() - base_url = unicode_to_str(base_url[0], response.encoding) if base_url \ - else unicode_to_str(response.url, response.encoding) + base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url links = [] for location in self.locations: diff --git a/scrapy/contrib/linkextractors/lxmlparser.py b/scrapy/contrib/linkextractors/lxmlparser.py index 390e3a304..27cd0697a 100644 --- a/scrapy/contrib/linkextractors/lxmlparser.py +++ b/scrapy/contrib/linkextractors/lxmlparser.py @@ -29,7 +29,7 @@ class LxmlLinkExtractor(object): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib/linkextractors/regex.py b/scrapy/contrib/linkextractors/regex.py index 08e1e4526..1de044df3 100644 --- a/scrapy/contrib/linkextractors/regex.py +++ b/scrapy/contrib/linkextractors/regex.py @@ -16,8 +16,9 @@ def clean_link(link_text): class RegexLinkExtractor(SgmlLinkExtractor): """High performant link extractor""" + def _extract_links(self, response_text, response_url, response_encoding): - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding)))) clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip() diff --git a/scrapy/contrib/linkextractors/sgml.py b/scrapy/contrib/linkextractors/sgml.py index d548626ab..9ec664bda 100644 --- a/scrapy/contrib/linkextractors/sgml.py +++ b/scrapy/contrib/linkextractors/sgml.py @@ -28,7 +28,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser): links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links ret = [] - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url for link in links: link.url = urljoin_rfc(base_url, link.url, response_encoding) link.url = safe_url_string(link.url, response_encoding) diff --git a/scrapy/contrib_exp/crawlspider/reqext.py b/scrapy/contrib_exp/crawlspider/reqext.py index bb7318f79..e23e78082 100644 --- a/scrapy/contrib_exp/crawlspider/reqext.py +++ b/scrapy/contrib_exp/crawlspider/reqext.py @@ -30,7 +30,7 @@ class BaseSgmlRequestExtractor(FixedSGMLParser): self.feed(response_text) self.close() - base_url = self.base_url if self.base_url else response_url + base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url self._make_absolute_urls(base_url, response_encoding) self._fix_link_text_encoding(response_encoding) diff --git a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py index d0d0d1b5f..0259b30fb 100644 --- a/scrapy/tests/test_contrib_exp_crawlspider_reqext.py +++ b/scrapy/tests/test_contrib_exp_crawlspider_reqext.py @@ -50,20 +50,39 @@ class RequestExtractorTest(AbstractRequestExtractorTest): ) def test_base_url(self): + reqx = BaseSgmlRequestExtractor() + html = """Page title<title> <base href="http://otherdomain.com/base/" /> <body><p><a href="item/12.html">Item 12</a></p> </body></html>""" - response = HtmlResponse("http://example.org/somepage/index.html", - body=html) - reqx = BaseSgmlRequestExtractor() + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('http://otherdomain.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) - self.failUnless( - self._requests_equals(reqx.extract_requests(response), - [ Request('http://otherdomain.com/base/item/12.html', - meta={'link_text': 'Item 12'}) ] - ) - ) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title> + <base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://example.org/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) + + # base url has no scheme + html = """<html><head><title>Page title<title> + <base href="//noscheme.com/base/" /> + <body><p><a href="item/12.html">Item 12</a></p> + </body></html>""" + response = HtmlResponse("https://example.org/p/index.html", body=html) + reqs = reqx.extract_requests(response) + self.failUnless(self._requests_equals( \ + [Request('https://noscheme.com/base/item/12.html', \ + meta={'link_text': 'Item 12'})], reqs), reqs) def test_extraction_encoding(self): #TODO: use own fixtures diff --git a/scrapy/tests/test_contrib_linkextractors.py b/scrapy/tests/test_contrib_linkextractors.py index 4a60b8a2d..65ffad714 100644 --- a/scrapy/tests/test_contrib_linkextractors.py +++ b/scrapy/tests/test_contrib_linkextractors.py @@ -35,6 +35,20 @@ class LinkExtractorTestCase(unittest.TestCase): self.assertEqual(lx.extract_links(response), [Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')]) + # base url is an absolute path and relative to host + html = """<html><head><title>Page title<title><base href="/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://example.org/item/12.html', text='Item 12')]) + + # base url has no scheme + html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" /> + <body><p><a href="item/12.html">Item 12</a></p></body></html>""" + response = HtmlResponse("https://example.org/somepage/index.html", body=html) + self.assertEqual(lx.extract_links(response), + [Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')]) + def test_extraction_encoding(self): body = get_testdata('link_extractor', 'linkextractor_noenc.html') response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})