some additional checks on using unicode url/body in Request/Response objects

scrapy/http/request/__init__.py

@@ -49,9 +49,14 @@ class Request(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if isinstance(url, basestring):
-            decoded_url = url if isinstance(url, unicode) else url.decode(self.encoding)
-            self._url = safe_url_string(decoded_url, self.encoding)
+        if isinstance(url, str):
+            self._url = safe_url_string(url)
+        elif isinstance(url, unicode):
+            if self.encoding is None:
+                raise TypeError('Cannot convert unicode url - %s has no encoding' %
+                    type(self).__name__)
+            unicode_url = url if isinstance(url, unicode) else url.decode(self.encoding)
+            self._url = safe_url_string(unicode_url, self.encoding)
         else:
             raise TypeError('Request url must be str or unicode, got %s:' % type(url).__name__)
 
@@ -64,6 +69,9 @@ class Request(object_ref):
         if isinstance(body, str):
             self._body = body
         elif isinstance(body, unicode):
+            if self.encoding is None:
+                raise TypeError('Cannot convert unicode body - %s has no encoding' %
+                    type(self).__name__)
             self._body = body.encode(self.encoding)
         elif body is None:
             self._body = ''

scrapy/http/response/text.py

@@ -29,6 +29,9 @@ class TextResponse(Response):
 
     def _set_url(self, url):
         if isinstance(url, unicode):
+            if self.encoding is None:
+                raise TypeError('Cannot convert unicode url - %s has no encoding' %
+                    type(self).__name__)
             self._url = url.encode(self.encoding)
         else:
             super(TextResponse, self)._set_url(url)
@@ -36,10 +39,11 @@ class TextResponse(Response):
     url = property(_get_url, _set_url)
 
     def _set_body(self, body):
+        self._body = ''
         if isinstance(body, unicode):
-            if self._encoding is None:
-                raise TypeError("To instantiate a %s with unicode body you " \
-                    "must specify the encoding" % self.__class__.__name__)
+            if self.encoding is None:
+                raise TypeError('Cannot convert unicode body - %s has no encoding' %
+                    type(self).__name__)
             self._body = body.encode(self._encoding)
         else:
             super(TextResponse, self)._set_body(body)

scrapy/tests/test_http_response.py

@@ -138,6 +138,12 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r3.encoding, "latin1")
 
     def test_unicode_url(self):
+        # instantiate with unicode url without encoding
+        self.assertRaises(TypeError, self.response_class, u"http://www.example.com/")
+        # make sure urls are converted to str
+        resp = self.response_class(url=u"http://www.example.com/", encoding='utf-8')
+        assert isinstance(resp.url, str)
+
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='utf-8')
         self.assertEqual(resp.url, 'http://www.example.com/price/\xc2\xa3')
         resp = self.response_class(url=u"http://www.example.com/price/\xa3", encoding='latin-1')