parse authentication credentials from url for http and ftp schemes

scrapy/downloadermiddlewares/auth.py

@@ -0,0 +1,46 @@
+"""
+HTTP basic auth downloader middleware
+
+See documentation in docs/topics/downloader-middleware.rst
+"""
+
+from w3lib.http import basic_auth_header
+
+from scrapy import signals
+
+from six.moves.urllib.parse import urlparse
+
+
+class AuthMiddleware(object):
+    """Set Basic HTTP Authorization header
+    (http_user and http_pass spider class attributes)"""
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        o = cls()
+        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+        return o
+
+    def spider_opened(self, spider):
+        usr = getattr(spider, 'http_user', '')
+        pwd = getattr(spider, 'http_pass', '')
+        if usr or pwd:
+            self.auth = basic_auth_header(usr, pwd)
+
+    def process_request(self, request, spider):
+        auth = getattr(self, 'auth', None)
+        if auth and 'Authorization' not in request.headers:
+            request.headers['Authorization'] = auth
+
+        # credentials from url are supposed to override spider settings
+        url = urlparse(request.url)
+        if url.username and url.password:
+            if url.scheme.startswith('ftp'):
+                request.meta['ftp_user'] = url.username
+                request.meta['ftp_password'] = url.password
+            elif url.scheme.startswith('http'):
+                request.headers['Authorization'] = basic_auth_header(url.username, url.password)
+
+            # no credentials in new url
+            new_url = url.scheme + '://' + url.hostname + url.path
+            return request.replace(url=new_url)

scrapy/downloadermiddlewares/httpauth.py

@@ -1,31 +1,7 @@
-"""
-HTTP basic auth downloader middleware
+import warnings
+from scrapy.exceptions import ScrapyDeprecationWarning
+warnings.warn("Module `scrapy.downloadermiddleware.httpauth` is deprecated, "
+              "use `scrapy.downloadermiddlewares.auth` instead",
+              ScrapyDeprecationWarning)
 
-See documentation in docs/topics/downloader-middleware.rst
-"""
-
-from w3lib.http import basic_auth_header
-
-from scrapy import signals
-
-
-class HttpAuthMiddleware(object):
-    """Set Basic HTTP Authorization header
-    (http_user and http_pass spider class attributes)"""
-
-    @classmethod
-    def from_crawler(cls, crawler):
-        o = cls()
-        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
-        return o
-
-    def spider_opened(self, spider):
-        usr = getattr(spider, 'http_user', '')
-        pwd = getattr(spider, 'http_pass', '')
-        if usr or pwd:
-            self.auth = basic_auth_header(usr, pwd)
-
-    def process_request(self, request, spider):
-        auth = getattr(self, 'auth', None)
-        if auth and b'Authorization' not in request.headers:
-            request.headers[b'Authorization'] = auth
+from scrapy.downloadermiddlewares.auth import AuthMiddleware as HttpAuthMiddleware

scrapy/settings/default_settings.py

@@ -91,7 +91,7 @@ DOWNLOADER_MIDDLEWARES = {}
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
-    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
+    'scrapy.downloadermiddlewares.auth.AuthMiddleware': 300,
     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,

tests/test_downloadermiddleware_auth.py

@@ -1,7 +1,7 @@
 import unittest
 
 from scrapy.http import Request
-from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
+from scrapy.downloadermiddlewares.auth import AuthMiddleware
 from scrapy.spiders import Spider
 
 
@@ -10,10 +10,10 @@ class TestSpider(Spider):
     http_pass = 'bar'
 
 
-class HttpAuthMiddlewareTest(unittest.TestCase):
+class AuthMiddlewareTest(unittest.TestCase):
 
     def setUp(self):
-        self.mw = HttpAuthMiddleware()
+        self.mw = AuthMiddleware()
         self.spider = TestSpider('foo')
         self.mw.spider_opened(self.spider)