diff --git a/scrapy/downloadermiddlewares/auth.py b/scrapy/downloadermiddlewares/auth.py new file mode 100644 index 000000000..b832d569a --- /dev/null +++ b/scrapy/downloadermiddlewares/auth.py @@ -0,0 +1,46 @@ +""" +HTTP basic auth downloader middleware + +See documentation in docs/topics/downloader-middleware.rst +""" + +from w3lib.http import basic_auth_header + +from scrapy import signals + +from six.moves.urllib.parse import urlparse + + +class AuthMiddleware(object): + """Set Basic HTTP Authorization header + (http_user and http_pass spider class attributes)""" + + @classmethod + def from_crawler(cls, crawler): + o = cls() + crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) + return o + + def spider_opened(self, spider): + usr = getattr(spider, 'http_user', '') + pwd = getattr(spider, 'http_pass', '') + if usr or pwd: + self.auth = basic_auth_header(usr, pwd) + + def process_request(self, request, spider): + auth = getattr(self, 'auth', None) + if auth and 'Authorization' not in request.headers: + request.headers['Authorization'] = auth + + # credentials from url are supposed to override spider settings + url = urlparse(request.url) + if url.username and url.password: + if url.scheme.startswith('ftp'): + request.meta['ftp_user'] = url.username + request.meta['ftp_password'] = url.password + elif url.scheme.startswith('http'): + request.headers['Authorization'] = basic_auth_header(url.username, url.password) + + # no credentials in new url + new_url = url.scheme + '://' + url.hostname + url.path + return request.replace(url=new_url) diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py index 7aa7a62bc..78adc8706 100644 --- a/scrapy/downloadermiddlewares/httpauth.py +++ b/scrapy/downloadermiddlewares/httpauth.py @@ -1,31 +1,7 @@ -""" -HTTP basic auth downloader middleware +import warnings +from scrapy.exceptions import ScrapyDeprecationWarning +warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, " + "use `scrapy.downloadermiddlewares.auth` instead", + ScrapyDeprecationWarning) -See documentation in docs/topics/downloader-middleware.rst -""" - -from w3lib.http import basic_auth_header - -from scrapy import signals - - -class HttpAuthMiddleware(object): - """Set Basic HTTP Authorization header - (http_user and http_pass spider class attributes)""" - - @classmethod - def from_crawler(cls, crawler): - o = cls() - crawler.signals.connect(o.spider_opened, signal=signals.spider_opened) - return o - - def spider_opened(self, spider): - usr = getattr(spider, 'http_user', '') - pwd = getattr(spider, 'http_pass', '') - if usr or pwd: - self.auth = basic_auth_header(usr, pwd) - - def process_request(self, request, spider): - auth = getattr(self, 'auth', None) - if auth and b'Authorization' not in request.headers: - request.headers[b'Authorization'] = auth +from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 18d5ebbbb..a6cfca656 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {} DOWNLOADER_MIDDLEWARES_BASE = { # Engine side 'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100, - 'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300, + 'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300, 'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350, 'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400, 'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500, diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_auth.py similarity index 83% rename from tests/test_downloadermiddleware_httpauth.py rename to tests/test_downloadermiddleware_auth.py index 425a5cc79..78fd120be 100644 --- a/tests/test_downloadermiddleware_httpauth.py +++ b/tests/test_downloadermiddleware_auth.py @@ -1,7 +1,7 @@ import unittest from scrapy.http import Request -from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware +from scrapy.downloadermiddlewares.auth import AuthMiddleware from scrapy.spiders import Spider @@ -10,10 +10,10 @@ class TestSpider(Spider): http_pass = 'bar' -class HttpAuthMiddlewareTest(unittest.TestCase): +class AuthMiddlewareTest(unittest.TestCase): def setUp(self): - self.mw = HttpAuthMiddleware() + self.mw = AuthMiddleware() self.spider = TestSpider('foo') self.mw.spider_opened(self.spider)