diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 729b80b05..82cf507f7 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -206,7 +206,7 @@ class ScrapyAgent(object): agent = self._get_agent(request, timeout) # request details - url = to_bytes(urldefrag(request.url)[0]) + url = urldefrag(request.url)[0] method = to_bytes(request.method) headers = TxHeaders(request.headers) if isinstance(agent, self._TunnelingAgent): @@ -214,7 +214,8 @@ class ScrapyAgent(object): bodyproducer = _RequestBodyProducer(request.body) if request.body else None start_time = time() - d = agent.request(method, url, headers, bodyproducer) + d = agent.request( + method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed @@ -267,10 +268,8 @@ class ScrapyAgent(object): txresponse, body, flags = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) - url = to_unicode(url) respcls = responsetypes.from_args(headers=headers, url=url) - return respcls( - url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags) @implementer(IBodyProducer)