mirror of https://github.com/scrapy/scrapy.git
added support for instantiating TextResponse (or any subclass) with unicode urls, improved organization of request/response unittests
This commit is contained in:
Pablo Hoffman
parent
e2bd1be995
commit
4023914b10
|
|
@ -12,14 +12,14 @@ from scrapy.utils.trackref import object_ref
|
|||
|
||||
class Response(object_ref):
|
||||
|
||||
__slots__ = ['url', 'headers', 'status', '_body', 'request', '_meta', \
|
||||
__slots__ = ['_url', 'headers', 'status', '_body', 'request', '_meta', \
|
||||
'flags', '__weakref__']
|
||||
|
||||
def __init__(self, url, status=200, headers=None, body='', meta=None, flags=None):
|
||||
self.url = url
|
||||
self.headers = Headers(headers or {})
|
||||
self.status = int(status)
|
||||
self._set_body(body)
|
||||
self._set_url(url)
|
||||
self.request = None
|
||||
self.flags = [] if flags is None else list(flags)
|
||||
self._meta = dict(meta) if meta else None
|
||||
|
|
@ -30,6 +30,18 @@ class Response(object_ref):
|
|||
self._meta = {}
|
||||
return self._meta
|
||||
|
||||
def _get_url(self):
|
||||
return self._url
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, str):
|
||||
self._url = url
|
||||
else:
|
||||
raise TypeError('%s url must be str, got %s:' % (type(self).__name__, \
|
||||
type(url).__name__))
|
||||
|
||||
url = property(_get_url, _set_url)
|
||||
|
||||
def _get_body(self):
|
||||
return self._body
|
||||
|
||||
|
|
|
|||
|
|
@ -24,6 +24,17 @@ class TextResponse(Response):
|
|||
self._body_inferred_encoding = None
|
||||
super(TextResponse, self).__init__(url, status, headers, body, meta, flags)
|
||||
|
||||
def _get_url(self):
|
||||
return self._url
|
||||
|
||||
def _set_url(self, url):
|
||||
if isinstance(url, unicode):
|
||||
self._url = url.encode(self.encoding)
|
||||
else:
|
||||
super(TextResponse, self)._set_url(url)
|
||||
|
||||
url = property(_get_url, _set_url)
|
||||
|
||||
def _set_body(self, body):
|
||||
if isinstance(body, unicode):
|
||||
if self._encoding is None:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class XPathSelector(object_ref):
|
|||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
elif text:
|
||||
response = TextResponse(url=None, body=unicode_to_str(text), \
|
||||
response = TextResponse(url='about:blank', body=unicode_to_str(text), \
|
||||
encoding='utf-8')
|
||||
self.doc = Libxml2Document(response, factory=self._get_libxml2_doc)
|
||||
self.xmlNode = self.doc.xmlDoc
|
||||
|
|
|
|||
|
|
@ -8,15 +8,17 @@ from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Response
|
|||
|
||||
class RequestTest(unittest.TestCase):
|
||||
|
||||
request_class = Request
|
||||
|
||||
def test_init(self):
|
||||
# Request requires url in the constructor
|
||||
self.assertRaises(Exception, Request)
|
||||
self.assertRaises(Exception, self.request_class)
|
||||
|
||||
# url argument must be basestring
|
||||
self.assertRaises(TypeError, Request, 123)
|
||||
r = Request('http://www.example.com')
|
||||
self.assertRaises(TypeError, self.request_class, 123)
|
||||
r = self.request_class('http://www.example.com')
|
||||
|
||||
r = Request("http://www.example.com")
|
||||
r = self.request_class("http://www.example.com")
|
||||
assert isinstance(r.url, str)
|
||||
self.assertEqual(r.url, "http://www.example.com")
|
||||
self.assertEqual(r.method, "GET")
|
||||
|
|
@ -30,7 +32,7 @@ class RequestTest(unittest.TestCase):
|
|||
|
||||
meta = {"lala": "lolo"}
|
||||
headers = {"caca": "coco"}
|
||||
r = Request("http://www.example.com", meta=meta, headers=headers, body="a body")
|
||||
r = self.request_class("http://www.example.com", meta=meta, headers=headers, body="a body")
|
||||
|
||||
assert r.meta is not meta
|
||||
self.assertEqual(r.meta, meta)
|
||||
|
|
@ -41,8 +43,8 @@ class RequestTest(unittest.TestCase):
|
|||
# Different ways of setting headers attribute
|
||||
url = 'http://www.scrapy.org'
|
||||
headers = {'Accept':'gzip', 'Custom-Header':'nothing to tell you'}
|
||||
r = Request(url=url, headers=headers)
|
||||
p = Request(url=url, headers=r.headers)
|
||||
r = self.request_class(url=url, headers=headers)
|
||||
p = self.request_class(url=url, headers=r.headers)
|
||||
|
||||
self.assertEqual(r.headers, p.headers)
|
||||
self.assertFalse(r.headers is headers)
|
||||
|
|
@ -58,8 +60,8 @@ class RequestTest(unittest.TestCase):
|
|||
|
||||
def test_eq(self):
|
||||
url = 'http://www.scrapy.org'
|
||||
r1 = Request(url=url)
|
||||
r2 = Request(url=url)
|
||||
r1 = self.request_class(url=url)
|
||||
r2 = self.request_class(url=url)
|
||||
self.assertNotEqual(r1, r2)
|
||||
|
||||
set_ = set()
|
||||
|
|
@ -69,7 +71,7 @@ class RequestTest(unittest.TestCase):
|
|||
|
||||
def test_url(self):
|
||||
"""Request url tests"""
|
||||
r = Request(url="http://www.scrapy.org/path")
|
||||
r = self.request_class(url="http://www.scrapy.org/path")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/path")
|
||||
|
||||
# url quoting on attribute assign
|
||||
|
|
@ -79,9 +81,9 @@ class RequestTest(unittest.TestCase):
|
|||
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
|
||||
|
||||
# url quoting on creation
|
||||
r = Request(url="http://www.scrapy.org/blank%20space")
|
||||
r = self.request_class(url="http://www.scrapy.org/blank%20space")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
|
||||
r = Request(url="http://www.scrapy.org/blank space")
|
||||
r = self.request_class(url="http://www.scrapy.org/blank space")
|
||||
self.assertEqual(r.url, "http://www.scrapy.org/blank%20space")
|
||||
|
||||
# url coercion to string
|
||||
|
|
@ -89,24 +91,24 @@ class RequestTest(unittest.TestCase):
|
|||
self.assert_(isinstance(r.url, str))
|
||||
|
||||
# url encoding
|
||||
r1 = Request(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8")
|
||||
r2 = Request(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1")
|
||||
r1 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8")
|
||||
r2 = self.request_class(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1")
|
||||
self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3")
|
||||
self.assertEqual(r2.url, "http://www.scrapy.org/price/%A3")
|
||||
|
||||
def test_body(self):
|
||||
r1 = Request(url="http://www.example.com/")
|
||||
r1 = self.request_class(url="http://www.example.com/")
|
||||
assert r1.body == ''
|
||||
|
||||
r2 = Request(url="http://www.example.com/", body="")
|
||||
r2 = self.request_class(url="http://www.example.com/", body="")
|
||||
assert isinstance(r2.body, str)
|
||||
self.assertEqual(r2.encoding, 'utf-8') # default encoding
|
||||
|
||||
r3 = Request(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
|
||||
r3 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8')
|
||||
assert isinstance(r3.body, str)
|
||||
self.assertEqual(r3.body, "Price: \xc2\xa3100")
|
||||
|
||||
r4 = Request(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
|
||||
r4 = self.request_class(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1')
|
||||
assert isinstance(r4.body, str)
|
||||
self.assertEqual(r4.body, "Price: \xa3100")
|
||||
|
||||
|
|
@ -116,7 +118,7 @@ class RequestTest(unittest.TestCase):
|
|||
def somecallback():
|
||||
pass
|
||||
|
||||
r1 = Request("http://www.example.com", callback=somecallback)
|
||||
r1 = self.request_class("http://www.example.com", callback=somecallback)
|
||||
r1.meta['foo'] = 'bar'
|
||||
r2 = r1.copy()
|
||||
|
||||
|
|
@ -137,7 +139,7 @@ class RequestTest(unittest.TestCase):
|
|||
def test_copy_inherited_classes(self):
|
||||
"""Test Request children copies preserve their class"""
|
||||
|
||||
class CustomRequest(Request):
|
||||
class CustomRequest(self.request_class):
|
||||
pass
|
||||
|
||||
r1 = CustomRequest('example.com', 'http://www.example.com')
|
||||
|
|
@ -148,7 +150,7 @@ class RequestTest(unittest.TestCase):
|
|||
def test_replace(self):
|
||||
"""Test Request.replace() method"""
|
||||
hdrs = Headers({"key": "value"})
|
||||
r1 = Request("http://www.example.com")
|
||||
r1 = self.request_class("http://www.example.com")
|
||||
r2 = r1.replace(method="POST", body="New body", headers=hdrs)
|
||||
self.assertEqual(r1.url, r2.url)
|
||||
self.assertEqual((r1.method, r2.method), ("GET", "POST"))
|
||||
|
|
@ -156,7 +158,7 @@ class RequestTest(unittest.TestCase):
|
|||
self.assertEqual((r1.headers, r2.headers), ({}, hdrs))
|
||||
|
||||
# Empty attributes (which may fail if not compared properly)
|
||||
r3 = Request("http://www.example.com", meta={'a': 1}, dont_filter=True)
|
||||
r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True)
|
||||
r4 = r3.replace(url="http://www.example.com/2", body='', meta={}, dont_filter=False)
|
||||
self.assertEqual(r4.url, "http://www.example.com/2")
|
||||
self.assertEqual(r4.body, '')
|
||||
|
|
@ -165,23 +167,24 @@ class RequestTest(unittest.TestCase):
|
|||
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
for cls in [Request, FormRequest]:
|
||||
x = cls('http://www.example.com')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
x = self.request_class('http://www.example.com')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
|
||||
class FormRequestTest(unittest.TestCase):
|
||||
class FormRequestTest(RequestTest):
|
||||
|
||||
request_class = FormRequest
|
||||
|
||||
def test_empty_formdata(self):
|
||||
r1 = FormRequest("http://www.example.com", formdata={})
|
||||
r1 = self.request_class("http://www.example.com", formdata={})
|
||||
self.assertEqual(r1.body, '')
|
||||
|
||||
def test_default_encoding(self):
|
||||
# using default encoding (utf-8)
|
||||
data = {'one': 'two', 'price': '\xc2\xa3 100'}
|
||||
r2 = FormRequest("http://www.example.com", formdata=data)
|
||||
r2 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertEqual(r2.method, 'POST')
|
||||
self.assertEqual(r2.encoding, 'utf-8')
|
||||
self.assertEqual(r2.body, 'price=%C2%A3+100&one=two')
|
||||
|
|
@ -189,14 +192,14 @@ class FormRequestTest(unittest.TestCase):
|
|||
|
||||
def test_custom_encoding(self):
|
||||
data = {'price': u'\xa3 100'}
|
||||
r3 = FormRequest("http://www.example.com", formdata=data, encoding='latin1')
|
||||
r3 = self.request_class("http://www.example.com", formdata=data, encoding='latin1')
|
||||
self.assertEqual(r3.encoding, 'latin1')
|
||||
self.assertEqual(r3.body, 'price=%A3+100')
|
||||
|
||||
def test_multi_key_values(self):
|
||||
# using multiples values for a single key
|
||||
data = {'price': u'\xa3 100', 'colours': ['red', 'blue', 'green']}
|
||||
r3 = FormRequest("http://www.example.com", formdata=data)
|
||||
r3 = self.request_class("http://www.example.com", formdata=data)
|
||||
self.assertEqual(r3.body, 'colours=red&colours=blue&colours=green&price=%C2%A3+100')
|
||||
|
||||
def test_from_response_post(self):
|
||||
|
|
@ -208,7 +211,7 @@ class FormRequestTest(unittest.TestCase):
|
|||
</form>
|
||||
"""
|
||||
response = Response("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, callback=lambda x: x)
|
||||
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'}, callback=lambda x: x)
|
||||
self.assertEqual(r1.method, 'POST')
|
||||
self.assertEqual(r1.headers['Content-type'], 'application/x-www-form-urlencoded')
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
|
|
@ -227,7 +230,7 @@ class FormRequestTest(unittest.TestCase):
|
|||
</form>
|
||||
"""
|
||||
response = Response("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = FormRequest.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
r1 = self.request_class.from_response(response, formdata={'one': ['two', 'three'], 'six': 'seven'})
|
||||
self.assertEqual(r1.method, 'GET')
|
||||
self.assertEqual(urlparse(r1.url).hostname, "www.example.com")
|
||||
self.assertEqual(urlparse(r1.url).path, "/this/get.php")
|
||||
|
|
@ -245,7 +248,7 @@ class FormRequestTest(unittest.TestCase):
|
|||
</form>
|
||||
"""
|
||||
response = Response("http://www.example.com/this/list.html", body=respbody)
|
||||
r1 = FormRequest.from_response(response, formdata={'two': '2'})
|
||||
r1 = self.request_class.from_response(response, formdata={'two': '2'})
|
||||
fs = cgi.FieldStorage(StringIO(r1.body), r1.headers, environ={"REQUEST_METHOD": "POST"})
|
||||
self.assertEqual(fs['one'].value, '1')
|
||||
self.assertEqual(fs['two'].value, '2')
|
||||
|
|
@ -253,7 +256,7 @@ class FormRequestTest(unittest.TestCase):
|
|||
def test_from_response_errors_noform(self):
|
||||
respbody = """<html></html>"""
|
||||
response = Response("http://www.example.com/lala.html", body=respbody)
|
||||
self.assertRaises(ValueError, FormRequest.from_response, response)
|
||||
self.assertRaises(ValueError, self.request_class.from_response, response)
|
||||
|
||||
def test_from_response_errors_formnumber(self):
|
||||
respbody = """
|
||||
|
|
@ -264,7 +267,9 @@ class FormRequestTest(unittest.TestCase):
|
|||
</form>
|
||||
"""
|
||||
response = Response("http://www.example.com/lala.html", body=respbody)
|
||||
self.assertRaises(IndexError, FormRequest.from_response, response, formnumber=1)
|
||||
self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1)
|
||||
|
||||
# XXX: XmlRpcRequest doesn't respect original Request API. is this expected?
|
||||
|
||||
class XmlRpcRequestTest(unittest.TestCase):
|
||||
|
||||
|
|
|
|||
|
|
@ -3,19 +3,22 @@ import weakref
|
|||
|
||||
from scrapy.http import Response, TextResponse, HtmlResponse, XmlResponse, Headers
|
||||
|
||||
class ResponseTest(unittest.TestCase):
|
||||
|
||||
class BaseResponseTest(unittest.TestCase):
|
||||
|
||||
response_class = Response
|
||||
|
||||
def test_init(self):
|
||||
# Response requires url in the consturctor
|
||||
self.assertRaises(Exception, Response)
|
||||
self.assertTrue(isinstance(Response('http://example.com/'), Response))
|
||||
# body can be str or None but not ResponseBody
|
||||
self.assertTrue(isinstance(Response('http://example.com/', body=''), Response))
|
||||
self.assertTrue(isinstance(Response('http://example.com/', body='body'), Response))
|
||||
self.assertRaises(Exception, self.response_class)
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/'), self.response_class))
|
||||
# body can be str or None
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/', body=''), self.response_class))
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/', body='body'), self.response_class))
|
||||
# test presence of all optional parameters
|
||||
self.assertTrue(isinstance(Response('http://example.com/', headers={}, status=200, body=''), Response))
|
||||
self.assertTrue(isinstance(self.response_class('http://example.com/', headers={}, status=200, body=''), self.response_class))
|
||||
|
||||
r = Response("http://www.example.com")
|
||||
r = self.response_class("http://www.example.com")
|
||||
assert isinstance(r.url, str)
|
||||
self.assertEqual(r.url, "http://www.example.com")
|
||||
self.assertEqual(r.status, 200)
|
||||
|
|
@ -27,23 +30,23 @@ class ResponseTest(unittest.TestCase):
|
|||
meta = {"lala": "lolo"}
|
||||
headers = {"caca": "coco"}
|
||||
body = "a body"
|
||||
r = Response("http://www.example.com", meta=meta, headers=headers, body="a body")
|
||||
r = self.response_class("http://www.example.com", meta=meta, headers=headers, body=body)
|
||||
|
||||
assert r.meta is not meta
|
||||
self.assertEqual(r.meta, meta)
|
||||
assert r.headers is not headers
|
||||
self.assertEqual(r.headers["caca"], "coco")
|
||||
|
||||
r = Response("http://www.example.com", status=301)
|
||||
r = self.response_class("http://www.example.com", status=301)
|
||||
self.assertEqual(r.status, 301)
|
||||
r = Response("http://www.example.com", status='301')
|
||||
r = self.response_class("http://www.example.com", status='301')
|
||||
self.assertEqual(r.status, 301)
|
||||
self.assertRaises(ValueError, Response, "http://example.com", status='lala200')
|
||||
self.assertRaises(ValueError, self.response_class, "http://example.com", status='lala200')
|
||||
|
||||
def test_copy(self):
|
||||
"""Test Response copy"""
|
||||
|
||||
r1 = Response("http://www.example.com", body="Some body")
|
||||
r1 = self.response_class("http://www.example.com", body="Some body")
|
||||
r1.meta['foo'] = 'bar'
|
||||
r1.flags.append('cached')
|
||||
r2 = r1.copy()
|
||||
|
|
@ -66,7 +69,7 @@ class ResponseTest(unittest.TestCase):
|
|||
def test_copy_inherited_classes(self):
|
||||
"""Test Response children copies preserve their class"""
|
||||
|
||||
class CustomResponse(Response):
|
||||
class CustomResponse(self.response_class):
|
||||
pass
|
||||
|
||||
r1 = CustomResponse('http://www.example.com')
|
||||
|
|
@ -77,7 +80,7 @@ class ResponseTest(unittest.TestCase):
|
|||
def test_replace(self):
|
||||
"""Test Response.replace() method"""
|
||||
hdrs = Headers({"key": "value"})
|
||||
r1 = Response("http://www.example.com")
|
||||
r1 = self.response_class("http://www.example.com")
|
||||
r2 = r1.replace(status=301, body="New body", headers=hdrs)
|
||||
assert r1.body == ''
|
||||
self.assertEqual(r1.url, r2.url)
|
||||
|
|
@ -85,33 +88,19 @@ class ResponseTest(unittest.TestCase):
|
|||
self.assertEqual((r1.body, r2.body), ('', "New body"))
|
||||
self.assertEqual((r1.headers, r2.headers), ({}, hdrs))
|
||||
|
||||
r1 = TextResponse("http://www.example.com", body="hello", encoding="cp852")
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
|
||||
|
||||
assert isinstance(r2, TextResponse)
|
||||
self.assertEqual(r2.url, "http://www.example.com/other")
|
||||
self.assertEqual(r2.encoding, "cp852")
|
||||
self.assertEqual(r3.url, "http://www.example.com/other")
|
||||
self.assertEqual(r3.encoding, "latin1")
|
||||
|
||||
# Empty attributes (which may fail if not compared properly)
|
||||
r3 = Response("http://www.example.com", meta={'a': 1}, flags=['cached'])
|
||||
r3 = self.response_class("http://www.example.com", meta={'a': 1}, flags=['cached'])
|
||||
r4 = r3.replace(body='', meta={}, flags=[])
|
||||
self.assertEqual(r4.body, '')
|
||||
self.assertEqual(r4.meta, {})
|
||||
self.assertEqual(r4.flags, [])
|
||||
|
||||
def test_encoding(self):
|
||||
unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
|
||||
self.assertRaises(TypeError, Response, 'http://www.example.com', body=u'unicode body')
|
||||
|
||||
original_string = unicode_string.encode('cp1251')
|
||||
r1 = TextResponse('http://www.example.com', body=original_string, encoding='cp1251')
|
||||
|
||||
# check body_as_unicode
|
||||
self.assertTrue(isinstance(r1.body_as_unicode(), unicode))
|
||||
self.assertEqual(r1.body_as_unicode(), unicode_string)
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
x = self.response_class('http://www.example.com')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
def _assert_response_values(self, response, encoding, body):
|
||||
if isinstance(body, unicode):
|
||||
|
|
@ -126,11 +115,57 @@ class ResponseTest(unittest.TestCase):
|
|||
self.assertEqual(response.body, body_str)
|
||||
self.assertEqual(response.body_as_unicode(), body_unicode)
|
||||
|
||||
def test_text_response(self):
|
||||
r1 = TextResponse("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xc2\xa3")
|
||||
r2 = TextResponse("http://www.example.com", encoding='utf-8', body=u"\xa3")
|
||||
r3 = TextResponse("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
|
||||
r4 = TextResponse("http://www.example.com", body="\xa2\xa3")
|
||||
class ResponseText(BaseResponseTest):
|
||||
|
||||
def test_no_unicode_url(self):
|
||||
self.assertRaises(TypeError, self.response_class, u'http://www.example.com')
|
||||
|
||||
|
||||
class TextResponseTest(BaseResponseTest):
|
||||
|
||||
response_class = TextResponse
|
||||
|
||||
def test_replace(self):
|
||||
super(TextResponseTest, self).test_replace()
|
||||
r1 = self.response_class("http://www.example.com", body="hello", encoding="cp852")
|
||||
r2 = r1.replace(url="http://www.example.com/other")
|
||||
r3 = r1.replace(url="http://www.example.com/other", encoding="latin1")
|
||||
|
||||
assert isinstance(r2, self.response_class)
|
||||
self.assertEqual(r2.url, "http://www.example.com/other")
|
||||
self.assertEqual(r2.encoding, "cp852")
|
||||
self.assertEqual(r3.url, "http://www.example.com/other")
|
||||
self.assertEqual(r3.encoding, "latin1")
|
||||
|
||||
def test_unicode_url(self):
|
||||
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
|
||||
resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
resp = self.response_class(url="http://www.example.com/price/", encoding='utf-8')
|
||||
resp.url = u'http://www.example.com/price/\xa3'
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=utf-8"]})
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
|
||||
resp = self.response_class(u"http://www.example.com/price/\xa3", headers={"Content-type": ["text/html; charset=iso-8859-1"]})
|
||||
self.assertEqual(resp.url, 'http://www.example.com/price/\xa3')
|
||||
|
||||
def test_unicode_body(self):
|
||||
unicode_string = u'\u043a\u0438\u0440\u0438\u043b\u043b\u0438\u0447\u0435\u0441\u043a\u0438\u0439 \u0442\u0435\u043a\u0441\u0442'
|
||||
self.assertRaises(TypeError, self.response_class, 'http://www.example.com', body=u'unicode body')
|
||||
|
||||
original_string = unicode_string.encode('cp1251')
|
||||
r1 = self.response_class('http://www.example.com', body=original_string, encoding='cp1251')
|
||||
|
||||
# check body_as_unicode
|
||||
self.assertTrue(isinstance(r1.body_as_unicode(), unicode))
|
||||
self.assertEqual(r1.body_as_unicode(), unicode_string)
|
||||
|
||||
def test_encoding(self):
|
||||
r1 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=utf-8"]}, body="\xc2\xa3")
|
||||
r2 = self.response_class("http://www.example.com", encoding='utf-8', body=u"\xa3")
|
||||
r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body="\xa3")
|
||||
r4 = self.response_class("http://www.example.com", body="\xa2\xa3")
|
||||
|
||||
self.assertEqual(r1.headers_encoding(), "utf-8")
|
||||
self.assertEqual(r2.headers_encoding(), None)
|
||||
|
|
@ -144,28 +179,33 @@ class ResponseTest(unittest.TestCase):
|
|||
self._assert_response_values(r3, 'iso-8859-1', u"\xa3")
|
||||
|
||||
# TextResponse (and subclasses) must be passed a encoding when instantiating with unicode bodies
|
||||
self.assertRaises(TypeError, TextResponse, "http://www.example.com", body=u"\xa3")
|
||||
self.assertRaises(TypeError, self.response_class, "http://www.example.com", body=u"\xa3")
|
||||
|
||||
|
||||
class HtmlResponseTest(TextResponseTest):
|
||||
|
||||
response_class = HtmlResponse
|
||||
|
||||
def test_html_encoding(self):
|
||||
|
||||
body = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
|
||||
</head><body>Price: \xa3100</body></html>'
|
||||
"""
|
||||
r1 = HtmlResponse("http://www.example.com", body=body)
|
||||
r1 = self.response_class("http://www.example.com", body=body)
|
||||
self._assert_response_values(r1, 'iso-8859-1', body)
|
||||
|
||||
body = """<?xml version="1.0" encoding="iso-8859-1"?>
|
||||
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
|
||||
Price: \xa3100
|
||||
"""
|
||||
r2 = HtmlResponse("http://www.example.com", body=body)
|
||||
r2 = self.response_class("http://www.example.com", body=body)
|
||||
self._assert_response_values(r2, 'iso-8859-1', body)
|
||||
|
||||
# for conflicting declarations headers must take precedence
|
||||
body = """<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
</head><body>Price: \xa3100</body></html>'
|
||||
"""
|
||||
r3 = HtmlResponse("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
|
||||
r3 = self.response_class("http://www.example.com", headers={"Content-type": ["text/html; charset=iso-8859-1"]}, body=body)
|
||||
self._assert_response_values(r3, 'iso-8859-1', body)
|
||||
|
||||
# make sure replace() preserves the encoding of the original response
|
||||
|
|
@ -173,40 +213,38 @@ class ResponseTest(unittest.TestCase):
|
|||
r4 = r3.replace(body=body)
|
||||
self._assert_response_values(r4, 'iso-8859-1', body)
|
||||
|
||||
|
||||
|
||||
class XmlResponseTest(TextResponseTest):
|
||||
|
||||
response_class = XmlResponse
|
||||
|
||||
def test_xml_encoding(self):
|
||||
|
||||
body = "<xml></xml>"
|
||||
r1 = XmlResponse("http://www.example.com", body=body)
|
||||
r1 = self.response_class("http://www.example.com", body=body)
|
||||
# XXX: we may want to swtich default XmlResponse encoding to utf-8
|
||||
self._assert_response_values(r1, 'ascii', body)
|
||||
|
||||
body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
|
||||
r2 = XmlResponse("http://www.example.com", body=body)
|
||||
r2 = self.response_class("http://www.example.com", body=body)
|
||||
self._assert_response_values(r2, 'iso-8859-1', body)
|
||||
|
||||
# make sure replace() preserves the explicit encoding passed in the constructor
|
||||
body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
|
||||
r3 = XmlResponse("http://www.example.com", body=body, encoding='utf-8')
|
||||
r3 = self.response_class("http://www.example.com", body=body, encoding='utf-8')
|
||||
body2 = "New body"
|
||||
r4 = r3.replace(body=body2)
|
||||
self._assert_response_values(r4, 'utf-8', body2)
|
||||
|
||||
# make sure replace() rediscovers the encoding (if not given explicitly) when changing the body
|
||||
body = """<?xml version="1.0" encoding="iso-8859-1"?><xml></xml>"""
|
||||
r5 = XmlResponse("http://www.example.com", body=body)
|
||||
r5 = self.response_class("http://www.example.com", body=body)
|
||||
body2 = """<?xml version="1.0" encoding="utf-8"?><xml></xml>"""
|
||||
r6 = r5.replace(body=body2)
|
||||
self._assert_response_values(r5, 'iso-8859-1', body)
|
||||
self._assert_response_values(r6, 'utf-8', body2)
|
||||
|
||||
def test_weakref_slots(self):
|
||||
"""Check that classes are using slots and are weak-referenceable"""
|
||||
for cls in [Response, TextResponse, XmlResponse, HtmlResponse]:
|
||||
x = cls('http://www.example.com')
|
||||
weakref.ref(x)
|
||||
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
|
||||
x.__class__.__name__
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue