adding configurable encoding for httpproxy authentication

docs/topics/downloader-middleware.rst

@@ -951,6 +951,18 @@ Default: ``False``
 Whether the AjaxCrawlMiddleware will be enabled. You may want to
 enable it for :ref:`broad crawls <topics-broad-crawls>`.
 
+HttpProxyMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: HTTPPROXY_AUTH_ENCODING
+
+HTTPPROXY_AUTH_ENCODING
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``"latin-1"``
+
+The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
+
 
 .. _DBM: http://en.wikipedia.org/wiki/Dbm
 .. _anydbm: https://docs.python.org/2/library/anydbm.html

scrapy/downloadermiddlewares/httpproxy.py

@@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.python import to_bytes
 
+
 class HttpProxyMiddleware(object):
 
-    def __init__(self):
+    def __init__(self, auth_encoding='latin-1'):
+        self.auth_encoding = auth_encoding
         self.proxies = {}
         for type, url in getproxies().items():
             self.proxies[type] = self._get_proxy(url, type)
@@ -21,12 +23,19 @@ class HttpProxyMiddleware(object):
         if not self.proxies:
             raise NotConfigured
 
+    @classmethod
+    def from_crawler(cls, crawler):
+        auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
+        return cls(auth_encoding)
+
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
         if user:
-            user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)))
+            user_pass = to_bytes(
+                '%s:%s' % (unquote(user), unquote(password)),
+                encoding=self.auth_encoding)
             creds = base64.b64encode(user_pass).strip()
         else:
             creds = None

scrapy/settings/default_settings.py

@@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+HTTPPROXY_AUTH_ENCODING = 'latin-1'
+
 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
 
 ITEM_PIPELINES = {}

tests/test_downloadermiddleware_httpproxy.py

@@ -9,6 +9,7 @@ from scrapy.spiders import Spider
 
 spider = Spider('foo')
 
+
 class TestDefaultHeadersMiddleware(TestCase):
 
     failureException = AssertionError
@@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase):
         self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
         self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
 
+    def test_proxy_auth_encoding(self):
+        # utf-8 encoding
+        os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+        mw = HttpProxyMiddleware(auth_encoding='utf-8')
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
+
+        # default latin-1 encoding
+        mw = HttpProxyMiddleware(auth_encoding='latin-1')
+        req = Request('http://scrapytest.org')
+        assert mw.process_request(req, spider) is None
+        self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
+
     def test_proxy_already_seted(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
@@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase):
         assert mw.process_request(req, spider) is None
         assert 'proxy' in req.meta and req.meta['proxy'] is None
 
-
     def test_no_proxy(self):
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
@@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase):
         req = Request('http://noproxy.com')
         assert mw.process_request(req, spider) is None
         assert 'proxy' not in req.meta
-