diff --git a/scrapy/trunk/scrapy/tests/test_utils_response.py b/scrapy/trunk/scrapy/tests/test_utils_response.py
index 2e86baf1e..701bf60ea 100644
--- a/scrapy/trunk/scrapy/tests/test_utils_response.py
+++ b/scrapy/trunk/scrapy/tests/test_utils_response.py
@@ -1,20 +1,20 @@
import unittest
from scrapy.http.response import Response
-from scrapy.utils.response import body_or_str
+from scrapy.utils.response import body_or_str, get_base_url
class ResponseUtilsTest(unittest.TestCase):
dummy_response = Response(url='http://example.org/', body='dummy_response')
- def test_input(self):
+ def test_body_or_str_input(self):
self.assertTrue(isinstance(body_or_str(self.dummy_response), basestring))
self.assertTrue(isinstance(body_or_str('text'), basestring))
self.assertRaises(Exception, body_or_str, 2)
- def test_extraction(self):
+ def test_body_or_str_extraction(self):
self.assertEqual(body_or_str(self.dummy_response), 'dummy_response')
self.assertEqual(body_or_str('text'), 'text')
- def test_encoding(self):
+ def test_body_or_str_encoding(self):
self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=False), str))
self.assertTrue(isinstance(body_or_str(self.dummy_response, unicode=True), unicode))
@@ -24,5 +24,14 @@ class ResponseUtilsTest(unittest.TestCase):
self.assertTrue(isinstance(body_or_str(u'text', unicode=False), str))
self.assertTrue(isinstance(body_or_str(u'text', unicode=True), unicode))
+ def test_get_base_url(self):
+ response = Response(url='http://example.org', body="""\
+ \
+ Dummy\
+ blahablsdfsal&\
+ """)
+ self.assertEqual(get_base_url(response), 'http://example.org/something')
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/scrapy/trunk/scrapy/utils/response.py b/scrapy/trunk/scrapy/utils/response.py
index c221314d7..ede0250e1 100644
--- a/scrapy/trunk/scrapy/utils/response.py
+++ b/scrapy/trunk/scrapy/utils/response.py
@@ -3,6 +3,7 @@ This module provides some useful functions for working with
scrapy.http.Response objects
"""
+import re
from scrapy.http.response import Response
def body_or_str(obj, unicode=True):
@@ -13,3 +14,14 @@ def body_or_str(obj, unicode=True):
return obj.decode('utf-8') if unicode else obj
else:
return obj if unicode else obj.encode('utf-8')
+
+BASEURL_RE = re.compile(r'