diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3641da231..a97d5a696 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -951,6 +951,18 @@ Default: ``False`` Whether the AjaxCrawlMiddleware will be enabled. You may want to enable it for :ref:`broad crawls `. +HttpProxyMiddleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: HTTPPROXY_AUTH_ENCODING + +HTTPPROXY_AUTH_ENCODING +^^^^^^^^^^^^^^^^^^^^^^^ + +Default: ``"latin-1"`` + +The default encoding for proxy authentication on :class:`HttpProxyMiddleware`. + .. _DBM: http://en.wikipedia.org/wiki/Dbm .. _anydbm: https://docs.python.org/2/library/anydbm.html diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py index 8c3514fd0..b01bab76d 100644 --- a/scrapy/downloadermiddlewares/httpproxy.py +++ b/scrapy/downloadermiddlewares/httpproxy.py @@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.exceptions import NotConfigured from scrapy.utils.python import to_bytes + class HttpProxyMiddleware(object): - def __init__(self): + def __init__(self, auth_encoding='latin-1'): + self.auth_encoding = auth_encoding self.proxies = {} for type, url in getproxies().items(): self.proxies[type] = self._get_proxy(url, type) @@ -21,12 +23,19 @@ class HttpProxyMiddleware(object): if not self.proxies: raise NotConfigured + @classmethod + def from_crawler(cls, crawler): + auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING') + return cls(auth_encoding) + def _get_proxy(self, url, orig_type): proxy_type, user, password, hostport = _parse_proxy(url) proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', '')) if user: - user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password))) + user_pass = to_bytes( + '%s:%s' % (unquote(user), unquote(password)), + encoding=self.auth_encoding) creds = base64.b64encode(user_pass).strip() else: creds = None diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index b151933b6..44e74dc61 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm' HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy' HTTPCACHE_GZIP = False +HTTPPROXY_AUTH_ENCODING = 'latin-1' + ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager' ITEM_PIPELINES = {} diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py index 7676b2a00..2b26431a4 100644 --- a/tests/test_downloadermiddleware_httpproxy.py +++ b/tests/test_downloadermiddleware_httpproxy.py @@ -9,6 +9,7 @@ from scrapy.spiders import Spider spider = Spider('foo') + class TestDefaultHeadersMiddleware(TestCase): failureException = AssertionError @@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase): self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=') + def test_proxy_auth_encoding(self): + # utf-8 encoding + os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128' + mw = HttpProxyMiddleware(auth_encoding='utf-8') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz') + + # default latin-1 encoding + mw = HttpProxyMiddleware(auth_encoding='latin-1') + req = Request('http://scrapytest.org') + assert mw.process_request(req, spider) is None + self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'}) + self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=') + def test_proxy_already_seted(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase): assert mw.process_request(req, spider) is None assert 'proxy' in req.meta and req.meta['proxy'] is None - def test_no_proxy(self): os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128' mw = HttpProxyMiddleware() @@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase): req = Request('http://noproxy.com') assert mw.process_request(req, spider) is None assert 'proxy' not in req.meta -