RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added get_base_url to utils.response and tests 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40760
This commit is contained in:
elpolilla 2009-01-22 16:40:01 +00:00
parent 86923cc694
commit 4142fd0d5d
2 changed files with 25 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
scrapy/trunk/scrapy/tests/test_utils_response.py
View File

@ -1,20 +1,20 @@
import unittest
from scrapy.http.response import Response
from scrapy.utils.response import body_or_str
from scrapy.utils.response import body_or_str, get_base_url
class ResponseUtilsTest(unittest.TestCase):
dummy_response = Response(url='http://example.org/', body='dummy_response')
def test_input(self):
def test_body_or_str_input(self):
self.assertTrue(isinstance(body_or_str(self.dummy_response), basestring))
self.assertTrue(isinstance(body_or_str('text'), basestring))
self.assertRaises(Exception, body_or_str, 2)
def test_extraction(self):
def test_body_or_str_extraction(self):
self.assertEqual(body_or_str(self.dummy_response), 'dummy_response')
self.assertEqual(body_or_str('text'), 'text')
def test_encoding(self):
def test_body_or_str_encoding(self):
self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=False), str))
self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=True), unicode))
@ -24,5 +24,14 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertTrue(isinstance(body_or_str(u'text', unicode=False), str))
self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))
def test_get_base_url(self):
response = Response(url='http://example.org', body="""\
<html>\
<head><title>Dummy</title><base href='http://example.org/something' /></head>\
<body>blahablsdfsal&amp;</body>\
</html>""")
self.assertEqual(get_base_url(response), 'http://example.org/something')
if __name__ == "__main__":
unittest.main()

12
scrapy/trunk/scrapy/utils/response.py
View File

@ -3,6 +3,7 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
import re
from scrapy.http.response import Response
def body_or_str(obj, unicode=True):
@ -13,3 +14,14 @@ def body_or_str(obj, unicode=True):
return obj.decode('utf-8') if unicode else obj
else:
return obj if unicode else obj.encode('utf-8')
BASEURL_RE = re.compile(r'<base\s+href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']', re.I)
def get_base_url(response):
""" Return the base url of the given response used to resolve relative links. """
# re is used instead of xpath for eficiency reasons. In a quick
# benchmark using timeit we got (for 50 repetitions) 0.0017 seconds
# using re and 0.7452 using xpath
if 'base_url' not in response.cache:
match = BASEURL_RE.search(response.body.to_string()[0:4096])
response.cache['base_url'] = match.group(1) if match else response.url
return response.cache['base_url']