RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support relative url used in base tag. closes #148 

--HG--
extra : rebase_source : 1bff87c127a7e9d8d12c772b3068feb11eb5d97f
This commit is contained in:
Daniel Grana 2010-03-25 12:38:37 -03:00
parent 9ddcd1095d
commit 173e94386b
8 changed files with 49 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/contrib/linkextractors/htmlparser.py
View File

@ -26,7 +26,7 @@ class HtmlParserLinkExtractor(HTMLParser):
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
ret = []
base_url = self.base_url if self.base_url else response_url
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
for link in links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
link.url = safe_url_string(link.url, response_encoding)

3
scrapy/contrib/linkextractors/image.py
View File

@ -51,8 +51,7 @@ class HTMLImageLinkExtractor(object):
def extract_links(self, response):
xs = HtmlXPathSelector(response)
base_url = xs.select('//base/@href').extract()
base_url = unicode_to_str(base_url[0], response.encoding) if base_url \
else unicode_to_str(response.url, response.encoding)
base_url = urljoin_rfc(response.url, base_url[0]) if base_url else response.url
links = []
for location in self.locations:

2
scrapy/contrib/linkextractors/lxmlparser.py
View File

@ -29,7 +29,7 @@ class LxmlLinkExtractor(object):
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
ret = []
base_url = self.base_url if self.base_url else response_url
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
for link in links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
link.url = safe_url_string(link.url, response_encoding)

3
scrapy/contrib/linkextractors/regex.py
View File

@ -16,8 +16,9 @@ def clean_link(link_text):
class RegexLinkExtractor(SgmlLinkExtractor):
"""High performant link extractor"""
def _extract_links(self, response_text, response_url, response_encoding):
base_url = self.base_url if self.base_url else response_url
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
clean_url = lambda u: urljoin_rfc(base_url, remove_entities(clean_link(u.decode(response_encoding))))
clean_text = lambda t: replace_escape_chars(remove_tags(t.decode(response_encoding))).strip()

2
scrapy/contrib/linkextractors/sgml.py
View File

@ -28,7 +28,7 @@ class BaseSgmlLinkExtractor(FixedSGMLParser):
links = unique_list(self.links, key=lambda link: link.url) if self.unique else self.links
ret = []
base_url = self.base_url if self.base_url else response_url
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
for link in links:
link.url = urljoin_rfc(base_url, link.url, response_encoding)
link.url = safe_url_string(link.url, response_encoding)

2
scrapy/contrib_exp/crawlspider/reqext.py
View File

@ -30,7 +30,7 @@ class BaseSgmlRequestExtractor(FixedSGMLParser):
self.feed(response_text)
self.close()
base_url = self.base_url if self.base_url else response_url
base_url = urljoin_rfc(response_url, self.base_url) if self.base_url else response_url
self._make_absolute_urls(base_url, response_encoding)
self._fix_link_text_encoding(response_encoding)

37
scrapy/tests/test_contrib_exp_crawlspider_reqext.py
View File

@ -50,20 +50,39 @@ class RequestExtractorTest(AbstractRequestExtractorTest):
)
def test_base_url(self):
reqx = BaseSgmlRequestExtractor()
html = """<html><head><title>Page title<title>
<base href="http://otherdomain.com/base/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>"""
response = HtmlResponse("http://example.org/somepage/index.html",
body=html)
reqx = BaseSgmlRequestExtractor()
response = HtmlResponse("https://example.org/p/index.html", body=html)
reqs = reqx.extract_requests(response)
self.failUnless(self._requests_equals( \
[Request('http://otherdomain.com/base/item/12.html', \
meta={'link_text': 'Item 12'})], reqs), reqs)
self.failUnless(
self._requests_equals(reqx.extract_requests(response),
[ Request('http://otherdomain.com/base/item/12.html',
meta={'link_text': 'Item 12'}) ]
)
)
# base url is an absolute path and relative to host
html = """<html><head><title>Page title<title>
<base href="/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>"""
response = HtmlResponse("https://example.org/p/index.html", body=html)
reqs = reqx.extract_requests(response)
self.failUnless(self._requests_equals( \
[Request('https://example.org/item/12.html', \
meta={'link_text': 'Item 12'})], reqs), reqs)
# base url has no scheme
html = """<html><head><title>Page title<title>
<base href="//noscheme.com/base/" />
<body><p><a href="item/12.html">Item 12</a></p>
</body></html>"""
response = HtmlResponse("https://example.org/p/index.html", body=html)
reqs = reqx.extract_requests(response)
self.failUnless(self._requests_equals( \
[Request('https://noscheme.com/base/item/12.html', \
meta={'link_text': 'Item 12'})], reqs), reqs)
def test_extraction_encoding(self):
#TODO: use own fixtures

14
scrapy/tests/test_contrib_linkextractors.py
View File

@ -35,6 +35,20 @@ class LinkExtractorTestCase(unittest.TestCase):
self.assertEqual(lx.extract_links(response),
[Link(url='http://otherdomain.com/base/item/12.html', text='Item 12')])
# base url is an absolute path and relative to host
html = """<html><head><title>Page title<title><base href="/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
self.assertEqual(lx.extract_links(response),
[Link(url='https://example.org/item/12.html', text='Item 12')])
# base url has no scheme
html = """<html><head><title>Page title<title><base href="//noschemedomain.com/path/to/" />
<body><p><a href="item/12.html">Item 12</a></p></body></html>"""
response = HtmlResponse("https://example.org/somepage/index.html", body=html)
self.assertEqual(lx.extract_links(response),
[Link(url='https://noschemedomain.com/path/to/item/12.html', text='Item 12')])
def test_extraction_encoding(self):
body = get_testdata('link_extractor', 'linkextractor_noenc.html')
response_utf8 = HtmlResponse(url='http://example.com/utf8', body=body, headers={'Content-Type': ['text/html; charset=utf-8']})