diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index df8c0105c..28c173505 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -49,9 +49,14 @@ class Request(object_ref): return self._url def _set_url(self, url): - if isinstance(url, basestring): - decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding) - self._url = safe_url_string(decoded_url, self.encoding) + if isinstance(url, str): + self._url = safe_url_string(url) + elif isinstance(url, unicode): + if self.encoding is None: + raise TypeError('Cannot convert unicode url - %s has no encoding' % + type(self).__name__) + unicode_url = url if isinstance(url, unicode) else url.decode(self.encoding) + self._url = safe_url_string(unicode_url, self.encoding) else: raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__) @@ -64,6 +69,9 @@ class Request(object_ref): if isinstance(body, str): self._body = body elif isinstance(body, unicode): + if self.encoding is None: + raise TypeError('Cannot convert unicode body - %s has no encoding' % + type(self).__name__) self._body = body.encode(self.encoding) elif body is None: self._body = '' diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index ba99fd88e..a3b609943 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -29,6 +29,9 @@ class TextResponse(Response): def _set_url(self, url): if isinstance(url, unicode): + if self.encoding is None: + raise TypeError('Cannot convert unicode url - %s has no encoding' % + type(self).__name__) self._url = url.encode(self.encoding) else: super(TextResponse, self)._set_url(url) @@ -36,10 +39,11 @@ class TextResponse(Response): url = property(_get_url, _set_url) def _set_body(self, body): + self._body = '' if isinstance(body, unicode): - if self._encoding is None: - raise TypeError("To instantiate a %s with unicode body you " \ - "must specify the encoding" % self.__class__.__name__) + if self.encoding is None: + raise TypeError('Cannot convert unicode body - %s has no encoding' % + type(self).__name__) self._body = body.encode(self._encoding) else: super(TextResponse, self)._set_body(body) diff --git a/scrapy/tests/test_http_response.py b/scrapy/tests/test_http_response.py index fb49422d8..88febd253 100644 --- a/scrapy/tests/test_http_response.py +++ b/scrapy/tests/test_http_response.py @@ -138,6 +138,12 @@ class TextResponseTest(BaseResponseTest): self.assertEqual(r3.encoding, "latin1") def test_unicode_url(self): + # instantiate with unicode url without encoding + self.assertRaises(TypeError, self.response_class, u"http://www.example.com/") + # make sure urls are converted to str + resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8') + assert isinstance(resp.url, str) + resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8') self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3') resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')