Support percent-encoded delimiters in userinfo

scrapy/downloadermiddlewares/auth.py

@@ -3,7 +3,7 @@ HTTP/FTP Authorization downloader middleware
 
 See documentation in docs/topics/downloader-middleware.rst
 """
-from six.moves.urllib.parse import urlunparse
+from six.moves.urllib.parse import unquote, urlunparse
 
 from w3lib.http import basic_auth_header
 
@@ -22,6 +22,11 @@ def credstrip_url(parsed_url):
         parsed_url.fragment))
 
 
+def _unquote(s):
+    if s is not None:
+        return unquote(s)
+
+
 class AuthMiddleware(object):
     """
     Populate authorization credentials for HTTP and FTP requests.
@@ -56,7 +61,8 @@ class AuthMiddleware(object):
             new_url = None
             # credentials from URL override spider attributes
             if url.username or url.password:
-                auth = basic_auth_header(url.username, url.password)
+                auth = basic_auth_header(_unquote(url.username),
+                                         _unquote(url.password))
 
                 # no credentials in new url
                 new_url = credstrip_url(url)
@@ -72,8 +78,8 @@ class AuthMiddleware(object):
         elif url.scheme.startswith('ftp'):
             if url.username or url.password:
                 # priorly set credentials take precedence
-                request.meta.setdefault('ftp_user', url.username)
-                request.meta.setdefault('ftp_password', url.password)
+                request.meta.setdefault('ftp_user', _unquote(url.username))
+                request.meta.setdefault('ftp_password', _unquote(url.password))
 
                 # no credentials in new url
                 return request.replace(url=credstrip_url(url))

tests/test_downloadermiddleware_auth.py

@@ -145,6 +145,22 @@ class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest):
         self.assertEquals(req.meta['ftp_password'], 'password')
         self.assertEquals(new_req.url, 'ftp://scrapytest.org/')
 
+    def test_auth_from_ftp_url_encoded_delims_user(self):
+        req = Request('ftp://username%3A:password@scrapytest.org/')
+        new_req = self.mw.process_request(req, self.spider)
+        assert new_req is not None
+        self.assertEquals(req.meta['ftp_user'], 'username:')
+        self.assertEquals(req.meta['ftp_password'], 'password')
+        self.assertEquals(new_req.url, 'ftp://scrapytest.org/')
+
+    def test_auth_from_ftp_url_encoded_delims_password(self):
+        req = Request('ftp://username:pass%40word@scrapytest.org/')
+        new_req = self.mw.process_request(req, self.spider)
+        assert new_req is not None
+        self.assertEquals(req.meta['ftp_user'], 'username')
+        self.assertEquals(req.meta['ftp_password'], 'pass@word')
+        self.assertEquals(new_req.url, 'ftp://scrapytest.org/')
+
     def test_auth_from_ftp_url_empty_user(self):
         req = Request('ftp://:password@scrapytest.org/')
         new_req = self.mw.process_request(req, self.spider)