diff --git a/scrapy/trunk/docs/ref/request-response.rst b/scrapy/trunk/docs/ref/request-response.rst index cd1e4ff77..7e416c822 100644 --- a/scrapy/trunk/docs/ref/request-response.rst +++ b/scrapy/trunk/docs/ref/request-response.rst @@ -21,7 +21,7 @@ generated the request. Request objects =============== -.. class:: Request(url, callback=None, method='GET', body=None, headers=None, cookies=None, meta=None, url_encoding='utf-8', dont_filter=None) +.. class:: Request(url, callback=None, method='GET', body=None, headers=None, cookies=None, meta=None, encoding='utf-8', dont_filter=None) A :class:`Request` object represents an HTTP request, which is usually generated in the Spider and executed by the Downloader, and thus generating @@ -44,9 +44,9 @@ Request objects ``cookies`` is a dict containing the request cookies - ``url_encoding`` is a string with the encoding of the url of this request. - The request URL will be percent encoded using this encoding before - downloading + ``encoding`` is a string with the encoding of this request. This encoding + will be used to percent-encode the URL and to convert the body to str (when + given as unicode). ``dont_filter`` is a boolean which indicates that this request should not be filtered by the scheduler. This is used when you want to perform an diff --git a/scrapy/trunk/scrapy/http/request.py b/scrapy/trunk/scrapy/http/request.py index 07b517e1f..444ef0861 100644 --- a/scrapy/trunk/scrapy/http/request.py +++ b/scrapy/trunk/scrapy/http/request.py @@ -18,28 +18,19 @@ from scrapy.utils.defer import chain_deferred class Request(object): def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, url_encoding='utf-8', dont_filter=None): - - self.encoding = url_encoding # this one has to be set first - self.set_url(url) + cookies=None, meta=None, encoding='utf-8', dont_filter=None): + self.encoding = encoding # this one has to be set first self.method = method.upper() + self.set_url(url) + self.set_body(body) - # body - if isinstance(body, dict): - body = urllib.urlencode(body) - self.body = body - - # callback / deferred if callable(callback): callback = defer.Deferred().addCallback(callback) self.deferred = callback or defer.Deferred() - # request cookies self.cookies = cookies or {} - # request headers - self.headers = Headers(headers or {}, encoding=url_encoding) - # dont_filter be filtered by scheduler + self.headers = Headers(headers or {}, encoding=encoding) self.dont_filter = dont_filter self.meta = {} if meta is None else dict(meta) @@ -57,6 +48,20 @@ class Request(object): self._url = Url(safe_url_string(decoded_url, self.encoding)) url = property(lambda x: x._url, set_url) + def set_body(self, body): + # TODO: move dict constructor to another Request class + if isinstance(body, dict): + self._body = urllib.urlencode(body) + elif body is None: + self._body = None + elif isinstance(body, str): + self._body = body + elif isinstance(body, unicode): + self._body = body.encode(self.encoding) + else: + raise TypeError("Request body must either str, unicode or None. Got: '%s'" % type(body).__name__) + body = property(lambda x: x._body, set_body) + def __str__(self): if self.method == 'GET': return "<%s>" % self.url diff --git a/scrapy/trunk/scrapy/tests/test_http_request.py b/scrapy/trunk/scrapy/tests/test_http_request.py index 2fdc56abb..60411ac6a 100644 --- a/scrapy/trunk/scrapy/tests/test_http_request.py +++ b/scrapy/trunk/scrapy/tests/test_http_request.py @@ -101,9 +101,26 @@ class RequestTest(unittest.TestCase): self.assert_(isinstance(r.url, str)) # url encoding - r = Request(url=u"http://www.scrapy.org/price/\xa3", url_encoding="utf-8") - self.assert_(isinstance(r.url, str)) - self.assertEqual(r.url, "http://www.scrapy.org/price/%C2%A3") + r1 = Request(url=u"http://www.scrapy.org/price/\xa3", encoding="utf-8") + r2 = Request(url=u"http://www.scrapy.org/price/\xa3", encoding="latin1") + self.assertEqual(r1.url, "http://www.scrapy.org/price/%C2%A3") + self.assertEqual(r2.url, "http://www.scrapy.org/price/%A3") + + def test_body(self): + r1 = Request(url="http://www.example.com/") + assert r1.body is None + + r2 = Request(url="http://www.example.com/", body="") + assert isinstance(r2.body, str) + self.assertEqual(r2.encoding, 'utf-8') # default encoding + + r3 = Request(url="http://www.example.com/", body=u"Price: \xa3100", encoding='utf-8') + assert isinstance(r3.body, str) + self.assertEqual(r3.body, "Price: \xc2\xa3100") + + r4 = Request(url="http://www.example.com/", body=u"Price: \xa3100", encoding='latin1') + assert isinstance(r4.body, str) + self.assertEqual(r4.body, "Price: \xa3100") def test_copy(self): """Test Request copy"""