diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py index b1242c69d..64239ab45 100644 --- a/scrapy/downloadermiddlewares/auth.py +++ b/scrapy/downloadermiddlewares/auth.py @@ -3,7 +3,7 @@ HTTP/FTP Authorization downloader middleware See documentation in docs/topics/downloader-middleware.rst """ -from six.moves.urllib.parse import urlunparse +from six.moves.urllib.parse import unquote, urlunparse from w3lib.http import basic_auth_header @@ -22,6 +22,11 @@ def credstrip_url(parsed_url): parsed_url.fragment)) +def _unquote(s): + if s is not None: + return unquote(s) + + class AuthMiddleware(object): """ Populate authorization credentials for HTTP and FTP requests. @@ -56,7 +61,8 @@ class AuthMiddleware(object): new_url = None # credentials from URL override spider attributes if url.username or url.password: - auth = basic_auth_header(url.username, url.password) + auth = basic_auth_header(_unquote(url.username), + _unquote(url.password)) # no credentials in new url new_url = credstrip_url(url) @@ -72,8 +78,8 @@ class AuthMiddleware(object): elif url.scheme.startswith('ftp'): if url.username or url.password: # priorly set credentials take precedence - request.meta.setdefault('ftp_user', url.username) - request.meta.setdefault('ftp_password', url.password) + request.meta.setdefault('ftp_user', _unquote(url.username)) + request.meta.setdefault('ftp_password', _unquote(url.password)) # no credentials in new url return request.replace(url=credstrip_url(url)) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index c58eabcb6..6f7a583ef 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -145,6 +145,22 @@ class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): self.assertEquals(req.meta['ftp_password'], 'password') self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url_encoded_delims_user(self): + req = Request('ftp://username%3A:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username:') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_encoded_delims_password(self): + req = Request('ftp://username:pass%40word@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], 'pass@word') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url_empty_user(self): req = Request('ftp://:password@scrapytest.org/') new_req = self.mw.process_request(req, self.spider)