py3: pass first http downloader test, simple crawler works now, yay!

scrapy/core/downloader/handlers/http11.py

@@ -19,6 +19,7 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
 from scrapy.utils.misc import load_object
+from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
 
 logger = logging.getLogger(__name__)
@@ -200,8 +201,8 @@ class ScrapyAgent(object):
         agent = self._get_agent(request, timeout)
 
         # request details
-        url = urldefrag(request.url)[0]
-        method = request.method
+        url = to_bytes(urldefrag(request.url)[0])
+        method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
         if isinstance(agent, self._TunnelingAgent):
             headers.removeHeader('Proxy-Authorization')
@@ -261,8 +262,10 @@ class ScrapyAgent(object):
         txresponse, body, flags = result
         status = int(txresponse.code)
         headers = Headers(txresponse.headers.getAllRawHeaders())
+        url = to_unicode(url)
         respcls = responsetypes.from_args(headers=headers, url=url)
-        return respcls(url=url, status=status, headers=headers, body=body, flags=flags)
+        return respcls(
+            url=url, status=status, headers=headers, body=body, flags=flags)
 
 
 @implementer(IBodyProducer)

scrapy/http/response/__init__.py

@@ -4,6 +4,7 @@ responses in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+import six
 from six.moves.urllib.parse import urljoin
 
 from scrapy.http.headers import Headers
@@ -34,7 +35,7 @@ class Response(object_ref):
         return self._url
 
     def _set_url(self, url):
-        if isinstance(url, str):
+        if isinstance(url, six.string_types):
             self._url = url
         else:
             raise TypeError('%s url must be str, got %s:' % (type(self).__name__,

tests/test_downloader_handlers.py

@@ -88,7 +88,7 @@ class FileTestCase(unittest.TestCase):
         def _test(response):
             self.assertEquals(response.url, request.url)
             self.assertEquals(response.status, 200)
-            self.assertEquals(response.body, '0123456789')
+            self.assertEquals(response.body, b'0123456789')
 
         request = Request(path_to_file_uri(self.tmpname + '^'))
         assert request.url.upper().endswith('%5E')
@@ -107,15 +107,15 @@ class HttpTestCase(unittest.TestCase):
     def setUp(self):
         name = self.mktemp()
         os.mkdir(name)
-        FilePath(name).child("file").setContent("0123456789")
+        FilePath(name).child("file").setContent(b"0123456789")
         r = static.File(name)
-        r.putChild("redirect", util.Redirect("/file"))
-        r.putChild("wait", ForeverTakingResource())
-        r.putChild("hang-after-headers", ForeverTakingResource(write=True))
-        r.putChild("nolength", NoLengthResource())
-        r.putChild("host", HostHeaderResource())
-        r.putChild("payload", PayloadResource())
-        r.putChild("broken", BrokenDownloadResource())
+        r.putChild(b"redirect", util.Redirect(b"/file"))
+        r.putChild(b"wait", ForeverTakingResource())
+        r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
+        r.putChild(b"nolength", NoLengthResource())
+        r.putChild(b"host", HostHeaderResource())
+        r.putChild(b"payload", PayloadResource())
+        r.putChild(b"broken", BrokenDownloadResource())
         self.site = server.Site(r, timeout=None)
         self.wrapper = WrappingFactory(self.site)
         self.port = reactor.listenTCP(0, self.wrapper, interface='127.0.0.1')
@@ -136,7 +136,7 @@ class HttpTestCase(unittest.TestCase):
         request = Request(self.getURL('file'))
         d = self.download_request(request, Spider('foo'))
         d.addCallback(lambda r: r.body)
-        d.addCallback(self.assertEquals, "0123456789")
+        d.addCallback(self.assertEquals, b"0123456789")
         return d
 
     def test_download_head(self):