diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index dda6a3d2a..8c3514fd0 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -9,7 +9,7 @@ from six.moves.urllib.parse import urlunparse from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured - +from scrapy.utils.python import to_bytes class HttpProxyMiddleware(object): @@ -26,7 +26,7 @@ class HttpProxyMiddleware(object): proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = '%s:%s' % (unquote(user), unquote(password)) + user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) creds = base64.b64encode(user_pass).strip() else: creds = None @@ -52,4 +52,4 @@ class HttpProxyMiddleware(object): creds, proxy = self.proxies[scheme] request.meta['proxy'] = proxy if creds: - request.headers['Proxy-Authorization'] = 'Basic ' + creds + request.headers['Proxy-Authorization'] = b'Basic ' + creds diff --git a/tests/py3-ignores.txt b/tests/py3-ignores.txt index 8c883ee3c..185a278fb 100644 --- a/tests/py3-ignores.txt +++ b/tests/py3-ignores.txt @@ -2,8 +2,6 @@ tests/test_closespider.py tests/test_exporters.py tests/test_linkextractors_deprecated.py tests/test_crawl.py -tests/test_downloadermiddleware_httpproxy.py -tests/test_downloadermiddleware_retry.py tests/test_mail.py tests/test_pipeline_files.py tests/test_pipeline_images.py @@ -23,8 +21,6 @@ scrapy/pipelines/files.py scrapy/linkextractors/sgml.py scrapy/linkextractors/regex.py scrapy/linkextractors/htmlparser.py -scrapy/downloadermiddlewares/retry.py -scrapy/downloadermiddlewares/httpproxy.py scrapy/downloadermiddlewares/cookies.py scrapy/extensions/statsmailer.py scrapy/extensions/memusage.py diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 5b9717a89..7676b2a00 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -52,7 +52,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjpwYXNz') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz') def test_proxy_auth_empty_passwd(self): os.environ['http_proxy'] = 'https://user:@proxy:3128' @@ -60,7 +60,7 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://scrapytest.org') assert mw.process_request(req, spider) is None self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) - self.assertEquals(req.headers.get('Proxy-Authorization'), 'Basic dXNlcjo=') + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index 20561e771..3de9399cf 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -21,20 +21,20 @@ class RetryTest(unittest.TestCase): def test_priority_adjust(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority < req.priority def test_404(self): req = Request('http://www.scrapytest.org/404') - rsp = Response('http://www.scrapytest.org/404', body='', status=404) + rsp = Response('http://www.scrapytest.org/404', body=b'', status=404) # dont retry 404s assert self.mw.process_response(req, rsp, self.spider) is rsp def test_dont_retry(self): req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry r = self.mw.process_response(req, rsp, self.spider) @@ -56,7 +56,7 @@ class RetryTest(unittest.TestCase): def test_503(self): req = Request('http://www.scrapytest.org/503') - rsp = Response('http://www.scrapytest.org/503', body='', status=503) + rsp = Response('http://www.scrapytest.org/503', body=b'', status=503) # first retry req = self.mw.process_response(req, rsp, self.spider)