diff --git a/docs/faq.rst b/docs/faq.rst index 415331515..d3efc09fc 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -149,7 +149,7 @@ See previous question. Can I use Basic HTTP Authentication in my spiders? -------------------------------------------------- -Yes, see :class:`~scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware`. +Yes, see :class:`~scrapy.downloadermiddlewares.auth.AuthMiddleware`. Why does Scrapy download pages in English instead of my native language? ------------------------------------------------------------------------ diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 15069e56e..01927e584 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -278,21 +278,30 @@ DownloadTimeoutMiddleware :reqmeta:`download_timeout` Request.meta key; this is supported even when DownloadTimeoutMiddleware is disabled. -HttpAuthMiddleware +AuthMiddleware ------------------ -.. module:: scrapy.downloadermiddlewares.httpauth - :synopsis: HTTP Auth downloader middleware +.. module:: scrapy.downloadermiddlewares.auth + :synopsis: HTTP/FTP Auth downloader middleware -.. class:: HttpAuthMiddleware +.. class:: AuthMiddleware - This middleware authenticates all requests generated from certain spiders - using `Basic access authentication`_ (aka. HTTP auth). + This middleware populates authentication credentials for HTTP and FTP requests. - To enable HTTP authentication from certain spiders, set the ``http_user`` - and ``http_pass`` attributes of those spiders. + To enable HTTP authentication (`Basic access authentication`_, aka. HTTP auth), + you have two options: - Example:: + - either set the ``http_user`` and ``http_pass`` attributes of the spider(s) + for which you need HTTP auth, + and these will be applied to all http(s):// requests, + - or, on a per-Request basis, include credentials using the standard URL + syntax of ``http://username:password@www.example.com/index.html`` + + To populate credentials for FTP requests, use URLs in the form of + ``ftp://user:password@www.example.com/document.txt``, which will + set the ``meta`` dict ``ftp_user`` and ``ftp_password`` values. + + Example of enabling HTTP Basic Auth for all HTTP requests:: from scrapy.spiders import CrawlSpider @@ -306,6 +315,15 @@ HttpAuthMiddleware .. _Basic access authentication: https://en.wikipedia.org/wiki/Basic_access_authentication +HttpAuthMiddleware +------------------ + +.. module:: scrapy.downloadermiddlewares.httpauth + :synopsis: HTTP Auth downloader middleware + +.. class:: HttpAuthMiddleware + + This middleware is deprecated and redirects to :class:`~.AuthMiddleware`. HttpCacheMiddleware ------------------- diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 8540308fe..f6493d155 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -457,7 +457,7 @@ Default:: { 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py index 211d2ecfc..64239ab45 100644 --- a/scrapy/downloadermiddlewares/auth.py +++ b/scrapy/downloadermiddlewares/auth.py @@ -1,19 +1,43 @@ """ -HTTP basic auth downloader middleware +HTTP/FTP Authorization downloader middleware See documentation in docs/topics/downloader-middleware.rst """ +from six.moves.urllib.parse import unquote, urlunparse from w3lib.http import basic_auth_header from scrapy import signals +from scrapy.utils.httpobj import urlparse_cached -from six.moves.urllib.parse import urlparse + +def credstrip_url(parsed_url): + """Strip username and password from an urlparse'd URL""" + return urlunparse(( + parsed_url.scheme, + parsed_url.netloc.split('@')[-1], + parsed_url.path, + parsed_url.params, + parsed_url.query, + parsed_url.fragment)) + + +def _unquote(s): + if s is not None: + return unquote(s) class AuthMiddleware(object): - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" + """ + Populate authorization credentials for HTTP and FTP requests. + + For http(s):// requests, set Basic HTTP Authorization header, + either from http_user and http_pass spider attributes, + or from URL netloc parsing. + + Also handle FTP credentials from ftp://user:password@... URLs, + populating request's meta accordingly. + """ @classmethod def from_crawler(cls, crawler): @@ -28,19 +52,34 @@ class AuthMiddleware(object): self.auth = basic_auth_header(usr, pwd) def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and 'Authorization' not in request.headers: - request.headers['Authorization'] = auth + url = urlparse_cached(request) + if url.scheme.startswith('http'): + # do not override Auth header set priorly + if 'Authorization' in request.headers: + return - # credentials from url are supposed to override spider settings - url = urlparse(request.url) - if url.username and url.password: - if url.scheme.startswith('ftp'): - request.meta['ftp_user'] = url.username - request.meta['ftp_password'] = url.password - elif url.scheme.startswith('http'): - request.headers['Authorization'] = basic_auth_header(url.username, url.password) + new_url = None + # credentials from URL override spider attributes + if url.username or url.password: + auth = basic_auth_header(_unquote(url.username), + _unquote(url.password)) - # no credentials in new url - new_url = url.scheme + '://' + request.url.split('@')[-1] - return request.replace(url=new_url) + # no credentials in new url + new_url = credstrip_url(url) + + else: + auth = getattr(self, 'auth', None) + + if auth: + request.headers['Authorization'] = auth + if new_url: + return request.replace(url=new_url) + + elif url.scheme.startswith('ftp'): + if url.username or url.password: + # priorly set credentials take precedence + request.meta.setdefault('ftp_user', _unquote(url.username)) + request.meta.setdefault('ftp_password', _unquote(url.password)) + + # no credentials in new url + return request.replace(url=credstrip_url(url)) diff --git a/tests/test_downloadermiddleware_auth.py b/tests/test_downloadermiddleware_auth.py index d5b5b4020..6f7a583ef 100644 --- a/tests/test_downloadermiddleware_auth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -10,6 +10,32 @@ class TestSpider(Spider): http_pass = 'bar' +class NoAuthTestSpider(Spider): + """A test spider that does not set http auth atttributes""" + + +class AuthMiddlewareNoAuthTest(unittest.TestCase): + + def setUp(self): + self.mw = AuthMiddleware() + self.spider = NoAuthTestSpider('bar') + self.mw.spider_opened(self.spider) + + def tearDown(self): + del self.mw + + def test_no_auth_http(self): + req = Request('http://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn('Authorization', req.headers) + + def test_no_auth_ftp(self): + req = Request('ftp://scrapytest.org/') + assert self.mw.process_request(req, self.spider) is None + self.assertNotIn('ftp_user', req.meta) + self.assertNotIn('ftp_password', req.meta) + + class AuthMiddlewareTest(unittest.TestCase): def setUp(self): @@ -20,6 +46,9 @@ class AuthMiddlewareTest(unittest.TestCase): def tearDown(self): del self.mw + +class AuthMiddlewareHttpAuthTest(AuthMiddlewareTest): + def test_auth(self): req = Request('http://scrapytest.org/') assert self.mw.process_request(req, self.spider) is None @@ -38,10 +67,120 @@ class AuthMiddlewareTest(unittest.TestCase): self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') self.assertEquals(new_req.url, 'http://scrapytest.org/') + def test_auth_from_https_url(self): + req = Request('https://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'https://scrapytest.org/') + + def test_auth_from_http_url_no_spider_attrs(self): + class AnotherTestSpider(Spider): + pass + req = Request('http://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, AnotherTestSpider('bar')) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_https_url_no_spider_attrs(self): + class AnotherTestSpider(Spider): + pass + req = Request('https://username:password@scrapytest.org/') + new_req = self.mw.process_request(req, AnotherTestSpider('bar')) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=') + self.assertEquals(new_req.url, 'https://scrapytest.org/') + + def test_auth_from_http_url_empty_pass(self): + req = Request('http://username:@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_http_url_pass_none(self): + req = Request('http://username@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic dXNlcm5hbWU6Tm9uZQ==') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + def test_auth_from_http_url_empty_user(self): + req = Request('http://:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.headers['Authorization'], b'Basic OnBhc3N3b3Jk') + self.assertEquals(new_req.url, 'http://scrapytest.org/') + + +class AuthMiddlewareFtpAuthTest(AuthMiddlewareTest): + + def test_no_auth_from_ftp_url_meta_unchanged(self): + usr, pwd = 'u', 'p' + req = Request('ftp://scrapytest.org/', + meta={"ftp_user": usr, "ftp_password": pwd}) + assert self.mw.process_request(req, self.spider) is None + self.assertEquals(req.meta['ftp_user'], usr) + self.assertEquals(req.meta['ftp_password'], pwd) + + def test_auth_from_ftp_url_meta_unchanged(self): + """Request's meta credentials are kept as-is, + but URL is stripped from credentials + """ + usr, pwd = 'u', 'p' + req = Request('ftp://username:password@scrapytest.org/', + meta={"ftp_user": usr, "ftp_password": pwd}) + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(new_req.meta['ftp_user'], usr) + self.assertEquals(new_req.meta['ftp_password'], pwd) + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + def test_auth_from_ftp_url(self): req = Request('ftp://username:password@scrapytest.org/') new_req = self.mw.process_request(req, self.spider) assert new_req is not None - self.assertIn('ftp_user', new_req.meta) - self.assertIn('ftp_password', new_req.meta) - self.assertNotIn('@', new_req.url) + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_encoded_delims_user(self): + req = Request('ftp://username%3A:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username:') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_encoded_delims_password(self): + req = Request('ftp://username:pass%40word@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], 'pass@word') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_empty_user(self): + req = Request('ftp://:password@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], '') + self.assertEquals(req.meta['ftp_password'], 'password') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_empty_pass(self): + req = Request('ftp://username:@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], '') + self.assertEquals(new_req.url, 'ftp://scrapytest.org/') + + def test_auth_from_ftp_url_pass_none(self): + req = Request('ftp://username@scrapytest.org/') + new_req = self.mw.process_request(req, self.spider) + assert new_req is not None + self.assertEquals(req.meta['ftp_user'], 'username') + self.assertEquals(req.meta['ftp_password'], None) + self.assertEquals(new_req.url, 'ftp://scrapytest.org/')