RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix handling of relative base urls in get_base_url util 

--HG--
extra : rebase_source : eb552219e6bf40bc0d2e35968c367105233b6ecc
This commit is contained in:
Daniel Grana 2010-03-25 15:50:34 -03:00
parent 1330697c3d
commit 996a1b3574
2 changed files with 18 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
scrapy/tests/test_utils_response.py
View File

@ -29,13 +29,29 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))
def test_get_base_url(self):
response = HtmlResponse(url='http://example.org', body="""\
response = HtmlResponse(url='https://example.org', body="""\
<html>\
<head><title>Dummy</title><base href='http://example.org/something' /></head>\
<body>blahablsdfsal&amp;</body>\
</html>""")
self.assertEqual(get_base_url(response), 'http://example.org/something')
# relative url with absolute path
response = HtmlResponse(url='https://example.org', body="""\
<html>\
<head><title>Dummy</title><base href='/absolutepath' /></head>\
<body>blahablsdfsal&amp;</body>\
</html>""")
self.assertEqual(get_base_url(response), 'https://example.org/absolutepath')
# no scheme url
response = HtmlResponse(url='https://example.org', body="""\
<html>\
<head><title>Dummy</title><base href='//noscheme.com/path' /></head>\
<body>blahablsdfsal&amp;</body>\
</html>""")
self.assertEqual(get_base_url(response), 'https://noscheme.com/path')
def test_get_meta_refresh(self):
body = """
<html>

2
scrapy/utils/response.py
View File

@ -33,7 +33,7 @@ def get_base_url(response):
""" Return the base url of the given response used to resolve relative links. """
if response not in _baseurl_cache:
match = BASEURL_RE.search(response.body_as_unicode()[0:4096])
_baseurl_cache[response] = match.group(1) if match else response.url
_baseurl_cache[response] = urljoin_rfc(response.url, match.group(1)) if match else response.url
return _baseurl_cache[response]
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', \