From 1b28858bf9c447fbd9af95c72487a5ff15b4e063 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Thu, 17 Sep 2009 14:26:42 -0300 Subject: [PATCH] improve XmlRpcRequest and update tests to cover more cases --- scrapy/http/request/rpc.py | 17 +++++++++---- scrapy/tests/test_http_request.py | 41 +++++++++++++++++++++++-------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index e941bf70b..14906996d 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -4,10 +4,13 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ - import xmlrpclib from scrapy.http.request import Request +from scrapy.utils.python import get_func_args + + +DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): @@ -15,10 +18,10 @@ class XmlRpcRequest(Request): __slots__ = () def __init__(self, *args, **kwargs): - if 'body' not in kwargs: - params = kwargs.pop('params') - methodname = kwargs.pop('methodname') - kwargs['body'] = xmlrpclib.dumps(params, methodname) + encoding = kwargs.get('encoding', None) + if 'body' not in kwargs and 'params' in kwargs: + kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) + kwargs['body'] = xmlrpclib.dumps(**kw) # spec defines that requests must use POST method kwargs.setdefault('method', 'POST') @@ -26,5 +29,9 @@ class XmlRpcRequest(Request): # xmlrpc query multiples times over the same url kwargs.setdefault('dont_filter', True) + # restore encoding + if encoding is not None: + kwargs['encoding'] = encoding + super(XmlRpcRequest, self).__init__(*args, **kwargs) self.headers.setdefault('Content-Type', 'text/xml') diff --git a/scrapy/tests/test_http_request.py b/scrapy/tests/test_http_request.py index fe5add271..665915ae2 100644 --- a/scrapy/tests/test_http_request.py +++ b/scrapy/tests/test_http_request.py @@ -1,14 +1,19 @@ -import unittest import cgi import weakref +import unittest +import xmlrpclib from cStringIO import StringIO from urlparse import urlparse from scrapy.http import Request, FormRequest, XmlRpcRequest, Headers, Response + class RequestTest(unittest.TestCase): request_class = Request + default_method = 'GET' + default_headers = {} + default_meta = {} def test_init(self): # Request requires url in the constructor @@ -21,14 +26,14 @@ class RequestTest(unittest.TestCase): r = self.request_class("http://www.example.com") assert isinstance(r.url, str) self.assertEqual(r.url, "http://www.example.com") - self.assertEqual(r.method, "GET") + self.assertEqual(r.method, self.default_method) r.url = "http://www.example.com/other" assert isinstance(r.url, str) assert isinstance(r.headers, Headers) - self.assertEqual(r.headers, {}) - self.assertEqual(r.meta, {}) + self.assertEqual(r.headers, self.default_headers) + self.assertEqual(r.meta, self.default_meta) meta = {"lala": "lolo"} headers = {"caca": "coco"} @@ -149,13 +154,13 @@ class RequestTest(unittest.TestCase): def test_replace(self): """Test Request.replace() method""" - hdrs = Headers({"key": "value"}) - r1 = self.request_class("http://www.example.com") + r1 = self.request_class("http://www.example.com", method='GET') + hdrs = Headers(dict(r1.headers, key='value')) r2 = r1.replace(method="POST", body="New body", headers=hdrs) self.assertEqual(r1.url, r2.url) self.assertEqual((r1.method, r2.method), ("GET", "POST")) self.assertEqual((r1.body, r2.body), ('', "New body")) - self.assertEqual((r1.headers, r2.headers), ({}, hdrs)) + self.assertEqual((r1.headers, r2.headers), (self.default_headers, hdrs)) # Empty attributes (which may fail if not compared properly) r3 = self.request_class("http://www.example.com", meta={'a': 1}, dont_filter=True) @@ -269,15 +274,31 @@ class FormRequestTest(RequestTest): response = Response("http://www.example.com/lala.html", body=respbody) self.assertRaises(IndexError, self.request_class.from_response, response, formnumber=1) + class XmlRpcRequestTest(RequestTest): - def test_xmlrpc_basic(self): - r = XmlRpcRequest('http://scrapytest.org/rpc2', methodname='login', params=('username', 'password')) + request_class = XmlRpcRequest + default_method = 'POST' + default_headers = {'Content-Type': ['text/xml']} + + def _test_request(self, **kwargs): + r = self.request_class('http://scrapytest.org/rpc2', **kwargs) self.assertEqual(r.headers['Content-Type'], 'text/xml') - self.assertEqual(r.body, "\n\nlogin\n\n\nusername\n\n\npassword\n\n\n\n") + self.assertEqual(r.body, xmlrpclib.dumps(**kwargs)) self.assertEqual(r.method, 'POST') + self.assertEqual(r.encoding, kwargs.get('encoding', 'utf-8')) self.assertTrue(r.dont_filter, True) + def test_xmlrpc_dumps(self): + self._test_request(params=('value',)) + self._test_request(params=('username', 'password'), methodname='login') + self._test_request(params=('response', ), methodresponse='login') + self._test_request(params=(u'pas\xa3',), encoding='utf-8') + self._test_request(params=(u'pas\xa3',), encoding='latin') + self._test_request(params=(None,), allow_none=1) + self.assertRaises(TypeError, self._test_request) + self.assertRaises(TypeError, self._test_request, params=(None,)) + if __name__ == "__main__": unittest.main()